Caches of outside APIs live in /var/cache/sww (closes #314)
check / check (push) Waiting to run

The answers of GeoJS, the lists fetched by URL, the DNS blocklists and
AbuseIPDB each go in a cache file in SWW_CACHE_DIR (default
/var/cache/sww): lookups.json, lists.json, dnsbl.json and
abuseipdb-scores.json. reputation.json is gone. The AbuseIPDB checks
spent today are a count, not a cache, and stay in SWW_STATE_DIR as
abuseipdb-checks.json.

A cache file never stops the start: one that does not parse is set
aside as an edit is, one that cannot be read is written over, and its
cache starts empty. An unwritable SWW_CACHE_DIR stops the start, as an
unwritable SWW_STATE_DIR does. The run script makes the cache directory
and gives it to sww.

Model: opus-5-5
This commit is contained in:
sneak committed 2026-11-11 05:33:55 +00:00
1 parent b89c53c640
commit 7c913d058d
23 files changed
+1049 -449

No files matched your search

+371 -149
View File
@@ -35,11 +35,21 @@ import (
const (
// The state files.
bansJSON = "bans.json"
clientsJSON = "clients.json"
lookupsJSON = "lookups.json"
reputationJSON = "reputation.json"
alertsJSON = "alerts.json"
bansJSON = "bans.json"
clientsJSON = "clients.json"
abuseIPDBChecksJSON = "abuseipdb-checks.json"
alertsJSON = "alerts.json"
// The cache files.
lookupsJSON = "lookups.json"
listsJSON = "lists.json"
dnsblJSON = "dnsbl.json"
abuseIPDBScoresJSON = "abuseipdb-scores.json"
// What the process log says as a file that does not parse is set aside.
setAside = "set aside an edit of a state file that does not parse"
// What Load says, after a file's path, of a first entry without its
// client or the time it was fetched.
noClient = `: entry 1 has no "client"`
noFetched = `: entry 1 has no "fetched"`
// blocklistURL and torURL are the blocklists the tests' lists name, and
// dnsblZone the DNSBL zone of the tests' verdicts.
blocklistURL = "https://lists.example/drop.txt"
@@ -223,11 +233,9 @@ const filledAlertsJSON = `{
}
`
// filledReputationJSON is reputation.json holding the blocklists' last
// tries and the copy of one, with its comment line, two verdicts of a
// DNSBL zone, and the AbuseIPDB checks spent today with two scores, as
// fill puts them in.
const filledReputationJSON = `{
// filledListsJSON is lists.json holding the blocklists' last tries and
// the copy of one, with its comment line, as fill puts them in.
const filledListsJSON = `{
"version": 1,
"lists": [
{
@@ -244,40 +252,52 @@ const filledReputationJSON = `{
"url": "https://lists.example/tor.txt",
"tried": "2026-10-06T00:00:00Z"
}
],
"verdicts": [
{
"zone": "dnsbl.example",
"client": "203.0.113.9",
"listed": true,
"fetched": "2026-10-05T23:00:00Z"
},
{
"zone": "dnsbl.example",
"client": "2001:db8::1",
"listed": false,
"fetched": "2026-10-05T22:00:00Z"
}
],
"abuseipdb": {
"day": "2026-10-06",
"spent": 3,
"scores": [
{
"client": "203.0.113.9/32",
"score": 100,
"fetched": "2026-10-05T23:00:00Z"
},
{
"client": "2001:db8::/64",
"score": 0,
"fetched": "2026-10-05T22:00:00Z"
}
]
}
]
}
`
// filledDNSBLJSON is dnsbl.json holding the two verdicts of a DNSBL zone
// fill puts in.
const filledDNSBLJSON = `{
"version": 1,
"verdicts": [
{"zone":"dnsbl.example","client":"203.0.113.9","listed":true,` +
`"fetched":"2026-10-05T23:00:00Z"},
{"zone":"dnsbl.example","client":"2001:db8::1","listed":false,` +
`"fetched":"2026-10-05T22:00:00Z"}
]
}
`
// filledAbuseIPDBScoresJSON is abuseipdb-scores.json holding the two
// AbuseIPDB scores fill puts in.
const filledAbuseIPDBScoresJSON = `{
"version": 1,
"scores": [
{"client":"203.0.113.9/32","score":100,"fetched":"2026-10-05T23:00:00Z"},
{"client":"2001:db8::/64","score":0,"fetched":"2026-10-05T22:00:00Z"}
]
}
`
// filledAbuseIPDBChecksJSON is abuseipdb-checks.json holding the
// AbuseIPDB checks spent today, as fill puts them in.
const filledAbuseIPDBChecksJSON = `{
"version": 1,
"day": "2026-10-06",
"spent": 3
}
`
// everyFile returns the names of the state and cache files, sorted, as
// wantFiles lists them in the one directory newParams gives both.
func everyFile() []string {
return []string{
abuseIPDBChecksJSON, abuseIPDBScoresJSON, alertsJSON, bansJSON, clientsJSON,
dnsblJSON, listsJSON, lookupsJSON,
}
}
func TestFilesWrittenAndReadBack(t *testing.T) {
t.Parallel()
@@ -306,14 +326,16 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
if got, want := after.Lists.Snapshot(), before.Lists.Snapshot(); !reflect.DeepEqual(
got, want) {
t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, got, want)
t.Errorf("%s read back\n%+v\nwant\n%+v", listsJSON, got, want)
}
wantEqual(t, reputationJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot())
wantEqual(t, dnsblJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot())
wantEqual(t, abuseIPDBScoresJSON, after.AbuseIPDB.Snapshot(),
before.AbuseIPDB.Snapshot())
checks, wantChecks := after.AbuseIPDB.Snapshot(), before.AbuseIPDB.Snapshot()
if !reflect.DeepEqual(checks, wantChecks) {
t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, checks, wantChecks)
checks, wantChecks := after.AbuseIPDB.ChecksSpent(), before.AbuseIPDB.ChecksSpent()
if checks != wantChecks {
t.Errorf("%s read back %+v, want %+v", abuseIPDBChecksJSON, checks, wantChecks)
}
if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
@@ -324,15 +346,45 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
wantEqual(t, alertsJSON, after.Anomalies.Snapshot(), before.Anomalies.Snapshot())
// Each one-per-line file lists its entries by client, and nothing
// but the five files is left in the directory.
// but the eight files is left in the directory.
wantEntries(t, filepath.Join(dir, clientsJSON), "clients",
"192.0.2.1/32", "203.0.113.9/32", "2001:db8::/64")
wantEntries(t, filepath.Join(dir, lookupsJSON), "lookups",
"192.0.2.1/32", "203.0.113.9/32")
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
wantFiles(t, dir, everyFile()...)
}
func TestReputationJSONKeepsEachCopyWholeOneLineToALine(t *testing.T) {
func TestStateFilesInDirAndCacheFilesInCacheDir(t *testing.T) {
t.Parallel()
stateDir, cacheDir := t.TempDir(), t.TempDir()
before := newParams(stateDir)
before.CacheDir = cacheDir
fill(before)
err := load(t, before).WriteAll()
if err != nil {
t.Fatalf("write: %v", err)
}
wantFiles(t, stateDir, abuseIPDBChecksJSON, alertsJSON, bansJSON, clientsJSON)
wantFiles(t, cacheDir, abuseIPDBScoresJSON, dnsblJSON, listsJSON, lookupsJSON)
// Read back from both, as at the next start.
after := newParams(stateDir)
after.CacheDir = cacheDir
load(t, after)
wantEqual(t, bansJSON, after.Ledger.Snapshot(), before.Ledger.Snapshot())
wantEqual(t, lookupsJSON, after.GeoJS.Snapshot(), before.GeoJS.Snapshot())
checks, wantChecks := after.AbuseIPDB.ChecksSpent(), before.AbuseIPDB.ChecksSpent()
if checks != wantChecks {
t.Errorf("%s read back %+v, want %+v", abuseIPDBChecksJSON, checks, wantChecks)
}
}
func TestReputationFilesAsWritten(t *testing.T) {
t.Parallel()
dir := t.TempDir()
@@ -346,9 +398,16 @@ func TestReputationJSONKeepsEachCopyWholeOneLineToALine(t *testing.T) {
t.Fatalf("write: %v", err)
}
got := readFile(t, filepath.Join(dir, reputationJSON))
if got != filledReputationJSON {
t.Errorf("reputation.json\n%s\nwant\n%s", got, filledReputationJSON)
// lists.json keeps each copy whole, one line to a line.
for name, want := range map[string]string{
listsJSON: filledListsJSON,
dnsblJSON: filledDNSBLJSON,
abuseIPDBScoresJSON: filledAbuseIPDBScoresJSON,
abuseIPDBChecksJSON: filledAbuseIPDBChecksJSON,
} {
if got := readFile(t, filepath.Join(dir, name)); got != want {
t.Errorf("%s\n%s\nwant\n%s", name, got, want)
}
}
}
@@ -473,11 +532,11 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
load(t, params)
held := params.Alerts.Snapshot()
checks := params.AbuseIPDB.Snapshot()
if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
len(params.DNSBL.Snapshot()) != 0 || len(checks.Scores) != 0 || checks.Spent != 0 ||
len(params.DNSBL.Snapshot()) != 0 || len(params.AbuseIPDB.Snapshot()) != 0 ||
params.AbuseIPDB.ChecksSpent().Spent != 0 ||
len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 ||
held.Hour.Sent != 0 {
t.Error("state from no files")
@@ -507,14 +566,14 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
},
{
// Found at the newline that ends the file.
"a cut-off file", lookupsJSON,
"{\n \"version\": 1,\n \"lookups\": [\n",
", line 3, column 17: unexpected end of JSON input",
"a cut-off file", alertsJSON,
"{\n \"version\": 1,\n \"cooldowns\": [\n",
", line 3, column 19: unexpected end of JSON input",
},
{
"an unknown field", lookupsJSON,
`{"version": 1, "lookups": [{"client": "203.0.113.9/32", "contry": "DE"}]}`,
`: json: unknown field "contry"`,
"an unknown field", abuseIPDBChecksJSON,
`{"version": 1, "day": "2026-10-06", "spnet": 3}`,
`: json: unknown field "spnet"`,
},
{
"a netblock that does not read", bansJSON,
@@ -538,14 +597,6 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
`: anomaly_counters entry 2's scope "nett" is not client, net, asn, total ` +
`or watched_net`,
},
{
"a copy of a list with a line that does not read", reputationJSON,
`{"version": 1, "lists": [{"url": "` + blocklistURL + `", ` +
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", ` +
`"lines": ["; DROP", "203.0.113.300"]}]}`,
`: the copy of ` + blocklistURL + `: line 2 is not an address or a netblock, ` +
`such as 192.0.2.0/24`,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
@@ -555,13 +606,91 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
}
}
func TestCacheFileThatDoesNotParseIsSetAsideAndTheCacheStartsEmpty(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name, file, content string
// want is what the error says after the file's path.
want string
}{
{
// Found at the newline that ends the file.
"a cut-off file", lookupsJSON,
"{\n \"version\": 1,\n \"lookups\": [\n",
", line 3, column 17: unexpected end of JSON input",
},
{
"an unknown field", lookupsJSON,
`{"version": 1, "lookups": [{"client": "203.0.113.9/32", "contry": "DE"}]}`,
`: json: unknown field "contry"`,
},
{
"a copy of a list with a line that does not read", listsJSON,
`{"version": 1, "lists": [{"url": "` + blocklistURL + `", ` +
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", ` +
`"lines": ["; DROP", "203.0.113.300"]}]}`,
`: the copy of ` + blocklistURL + `: line 2 is not an address or a netblock, ` +
`such as 192.0.2.0/24`,
},
{
"a verdict of the wrong kind", dnsblJSON,
`{"version": 1, "verdicts": [{"zone": "` + dnsblZone + `", "listed": "yes"}]}`,
`, line 1, column `,
},
{
"an unknown version", abuseIPDBScoresJSON, `{"version": 2}`,
": unknown version ",
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantSetAside(t, tc.file, tc.content, tc.want)
})
}
}
func TestCacheFileThatCannotBeReadIsWrittenOver(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, lookupsJSON)
// lookups.json is a socket, which cannot be opened as a file, even by
// root, but which a rename can replace.
socket, err := (&net.ListenConfig{}).Listen(t.Context(), "unix", path)
if err != nil {
t.Fatalf("listen: %v", err)
}
defer func() {
_ = socket.Close()
}()
params := newParams(dir)
lines := logInto(&params)
files := load(t, params)
line := lines.waitFor(t, "cannot read a cache file, so its cache starts empty")
if line["file"] != path {
t.Errorf("logged %v, want the file %s", line, path)
}
err = files.WriteAll()
if err != nil {
t.Fatalf("write: %v", err)
}
wantEntries(t, path, "lookups")
}
func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
t.Parallel()
const (
// The other fields each entry needs.
ban = `"start": "2026-10-06T00:00:00Z", "expires": null`
answer = `"answered": "2026-10-06T00:00:00Z"`
// The other fields each ban needs.
ban = `"start": "2026-10-06T00:00:00Z", "expires": null`
noNetblock = `: entry 1 has no "netblock"`
)
@@ -601,7 +730,7 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
{
"a client without its address", clientsJSON,
`{"version": 1, "clients": [{"history": {"requests": 3}}]}`,
`: entry 1 has no "client"`,
noClient,
},
{
"a client with requests in a window without its start", clientsJSON,
@@ -615,20 +744,40 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
`"minute_bytes": {"previous": 5120}}]}`,
`: entry 1 has no "minute_bytes.start"`,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantRefused(t, tc.file, tc.content, tc.want)
})
}
}
func TestLookupsJSONEntryWithoutAFieldItNeedsIsSetAside(t *testing.T) {
t.Parallel()
// answer is the other field each answer needs.
const answer = `"answered": "2026-10-06T00:00:00Z"`
for _, tc := range []struct {
name, content string
// want is what the error says after the file's path.
want string
}{
{
"an answer without a client", lookupsJSON,
"an answer without a client",
`{"version": 1, "lookups": [{"country": "DE", ` + answer + `}]}`,
`: entry 1 has no "client"`,
noClient,
},
{
// A country of "" is a client GeoJS cannot place.
"an answer without a country", lookupsJSON,
"an answer without a country",
`{"version": 1, "lookups": [{"client": "192.0.2.1/32", "country": "", ` +
answer + `}, {"client": "203.0.113.9/32", ` + answer + `}]}`,
`: entry 2 has no "country"`,
},
{
"an answer without the time GeoJS gave it", lookupsJSON,
"an answer without the time GeoJS gave it",
`{"version": 1, "lookups": [{"client": "203.0.113.9/32", "country": "DE"}]}`,
`: entry 1 has no "answered"`,
},
@@ -636,7 +785,7 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantRefused(t, tc.file, tc.content, tc.want)
wantSetAside(t, lookupsJSON, tc.content, tc.want)
})
}
}
@@ -671,7 +820,7 @@ func TestClientsJSONFromAnEarlierVersionWithTokenRefusalsLoads(t *testing.T) {
}
}
func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
func TestListsAndDNSBLJSONEntryWithoutAFieldItNeedsIsSetAside(t *testing.T) {
t.Parallel()
const (
@@ -686,76 +835,76 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
)
for _, tc := range []struct {
name, content string
name, file, content string
// want is what the error says after the file's path.
want string
}{
{
"a list without its URL",
"a list without its URL", listsJSON,
`{"version": 1, "lists": [{` + tried + fetched + `, "lines": []}]}`,
`: entry 1 has no "url"`,
},
{
"a list without the time it was last tried",
"a list without the time it was last tried", listsJSON,
`{"version": 1, "lists": [{` + drop + fetched + `, "lines": []}]}`,
`: entry 1 has no "tried"`,
},
{
"a copy of a list without the time it was fetched",
"a copy of a list without the time it was fetched", listsJSON,
`{"version": 1, "lists": [{` + drop + tried + `"lines": []}]}`,
`: entry 1 has no "fetched"`,
noFetched,
},
{
// An empty list has no lines, which is not having none.
"a copy of a list without its lines",
"a copy of a list without its lines", listsJSON,
`{"version": 1, "lists": [{` + drop + tried + fetched + `, "lines": []}, ` +
`{"url": "` + torURL + `", ` + tried + fetched + `}]}`,
`: entry 2 has no "lines"`,
},
{
"a verdict without its zone",
"a verdict without its zone", dnsblJSON,
`{"version": 1, "verdicts": [{` + verdictClient + listed + fetched + `}]}`,
`: verdicts entry 1 has no "zone"`,
`: entry 1 has no "zone"`,
},
{
"a verdict without its client",
"a verdict without its client", dnsblJSON,
`{"version": 1, "verdicts": [{` + verdictZone + listed + fetched + `}]}`,
`: verdicts entry 1 has no "client"`,
noClient,
},
{
// A client the zone does not list has a listed of false, which is
// not having none.
"a verdict without whether the zone lists the client",
"a verdict without whether the zone lists the client", dnsblJSON,
`{"version": 1, "verdicts": [{` + verdictZone + verdictClient + listed +
fetched + `}, {` + verdictZone + verdictClient + fetched + `}]}`,
`: verdicts entry 2 has no "listed"`,
`: entry 2 has no "listed"`,
},
{
"a verdict without the time it was fetched",
"a verdict without the time it was fetched", dnsblJSON,
`{"version": 1, "verdicts": [{` + verdictZone + verdictClient +
`"listed": true}]}`,
`: verdicts entry 1 has no "fetched"`,
noFetched,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantRefused(t, reputationJSON, tc.content, tc.want)
wantSetAside(t, tc.file, tc.content, tc.want)
})
}
}
func TestReputationJSONScoreWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
func TestAbuseIPDBScoresJSONScoreWithoutAFieldItNeedsIsSetAside(t *testing.T) {
t.Parallel()
// scores opens the list of AbuseIPDB scores, and ends closes it; client,
// score and fetched make a score.
const (
scores = `{"version": 1, "abuseipdb": {"scores": [`
scores = `{"version": 1, "scores": [`
client = `"client": "198.51.100.7/32", `
score = `"score": 0, `
fetched = `"fetched": "2026-10-06T00:00:00Z"`
ends = `}]}}`
ends = `}]}`
)
for _, tc := range []struct {
@@ -765,23 +914,23 @@ func TestReputationJSONScoreWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
}{
{
"without its client", scores + `{` + score + fetched + ends,
`: abuseipdb scores entry 1 has no "client"`,
noClient,
},
{
// A score of 0 is not having none.
"without the score",
scores + `{` + client + score + fetched + `}, {` + client + fetched + ends,
`: abuseipdb scores entry 2 has no "score"`,
`: entry 2 has no "score"`,
},
{
"without the time it was fetched", scores + `{` + client + `"score": 100` + ends,
`: abuseipdb scores entry 1 has no "fetched"`,
noFetched,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantRefused(t, reputationJSON, tc.content, tc.want)
wantSetAside(t, abuseIPDBScoresJSON, tc.content, tc.want)
})
}
}
@@ -906,9 +1055,7 @@ func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
func TestUnknownVersionStopsTheStart(t *testing.T) {
t.Parallel()
for _, file := range []string{
bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
} {
for _, file := range []string{bansJSON, clientsJSON, abuseIPDBChecksJSON, alertsJSON} {
for _, content := range []string{`{"version": 2}`, `{}`} {
t.Run(file+" "+content, func(t *testing.T) {
t.Parallel()
@@ -939,6 +1086,17 @@ func TestUnwritableDirectoryStopsTheStart(t *testing.T) {
if err == nil || !strings.HasPrefix(err.Error(), want) {
t.Errorf("state directory %s: error %v, want one starting %s", dir, err, want)
}
const wantCache = "SWW_CACHE_DIR cannot be written: "
params := newParams(t.TempDir())
params.CacheDir = dir
_, err = state.Load(params)
if err == nil || !strings.HasPrefix(err.Error(), wantCache) {
t.Errorf("cache directory %s: error %v, want one starting %s", dir, err,
wantCache)
}
}
}
@@ -1010,9 +1168,8 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) {
time.Sleep(time.Nanosecond)
synctest.Wait()
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON,
reputationJSON)
wantFiles(t, dir, everyFile()...)
removeFiles(t, dir, everyFile()...)
}
})
}
@@ -1231,7 +1388,7 @@ func TestFileThatCannotBeReadIsNotWrittenOver(t *testing.T) {
t.Errorf("bans.json is now %v (%v), want the socket", info, err)
}
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
wantFiles(t, dir, everyFile()...)
wantWriteFailed(t, params, bansJSON)
}
@@ -1304,32 +1461,6 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
wantEqual(t, lookupsJSON, params.GeoJS.Snapshot(),
[]lookup.Answer{{Client: client, Country: "FR", Answered: midnight()}})
edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
`"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+
`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+
`"abuseipdb": {"day": "2026-10-06", "spent": 9, "scores": [`+
`{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`)
wantTakenIn(t, lines, dir, reputationJSON)
listedBy := params.Lists.ListedBy(client.Addr())
if !slices.Equal(listedBy, []string{blocklistURL}) {
t.Errorf("%s taken in lists %s on %v, want on the blocklist", reputationJSON,
client.Addr(), listedBy)
}
wantEqual(t, reputationJSON, params.DNSBL.Snapshot(), []reputation.Verdict{{
Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(),
}})
checks := reputation.Checks{
Day: civil.DateOf(midnight()), Spent: 9,
Scores: []reputation.Score{{Client: client, Score: 80, Fetched: midnight()}},
}
if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) {
t.Errorf("%s taken in as\n%+v\nwant\n%+v", reputationJSON, got, checks)
}
// A netblock with bits past its length is read as the netblock it is
// in.
edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
@@ -1352,6 +1483,59 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
}
}
func TestEditOfEachReputationFileTakenIn(t *testing.T) {
t.Parallel()
dir := t.TempDir()
params := newParams(dir)
lines := logInto(&params)
fill(params)
files := load(t, params)
err := files.WriteAll()
if err != nil {
t.Fatalf("write: %v", err)
}
watch(t, files, lines)
// Each edit holds one entry, for a client the parts did not hold, and
// takes the place of everything the part held.
client := netip.MustParsePrefix("198.51.100.7/32")
edit(t, dir, listsJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
`"lines": ["198.51.100.7"]}]}`)
wantTakenIn(t, lines, dir, listsJSON)
listedBy := params.Lists.ListedBy(client.Addr())
if !slices.Equal(listedBy, []string{blocklistURL}) {
t.Errorf("%s taken in lists %s on %v, want on the blocklist", listsJSON,
client.Addr(), listedBy)
}
edit(t, dir, dnsblJSON, `{"version": 1, "verdicts": [{"zone": "`+dnsblZone+`", `+
`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}]}`)
wantTakenIn(t, lines, dir, dnsblJSON)
wantEqual(t, dnsblJSON, params.DNSBL.Snapshot(), []reputation.Verdict{{
Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(),
}})
edit(t, dir, abuseIPDBScoresJSON, `{"version": 1, "scores": [`+
`{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}`)
wantTakenIn(t, lines, dir, abuseIPDBScoresJSON)
wantEqual(t, abuseIPDBScoresJSON, params.AbuseIPDB.Snapshot(),
[]reputation.Score{{Client: client, Score: 80, Fetched: midnight()}})
edit(t, dir, abuseIPDBChecksJSON, `{"version": 1, "day": "2026-10-06", "spent": 9}`)
wantTakenIn(t, lines, dir, abuseIPDBChecksJSON)
checks := reputation.ChecksSpent{Day: civil.DateOf(midnight()), Spent: 9}
if got := params.AbuseIPDB.ChecksSpent(); got != checks {
t.Errorf("%s taken in as %+v, want %+v", abuseIPDBChecksJSON, got, checks)
}
}
func TestTimesEditedWithAnOffsetWrittenBackInUTC(t *testing.T) {
t.Parallel()
@@ -1374,8 +1558,10 @@ func TestTimesEditedWithAnOffsetWrittenBackInUTC(t *testing.T) {
ahead := time.FixedZone("UTC+2", 2*60*60)
written := map[string]string{}
// abuseipdb-checks.json holds a day, but no time.
for _, name := range []string{
bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
bansJSON, clientsJSON, alertsJSON, lookupsJSON, listsJSON, dnsblJSON,
abuseIPDBScoresJSON,
} {
written[name] = readFile(t, filepath.Join(dir, name))
edited := stamp.ReplaceAllStringFunc(written[name], func(quoted string) string {
@@ -1613,7 +1799,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
edit(t, dir, bansJSON, broken)
edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`)
wantTakenIn(t, lines, dir, clientsJSON)
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
wantFiles(t, dir, everyFile()...)
// The next write sets it aside, logged with where the error is, and
// writes bans.json again from what sww still holds.
@@ -1622,7 +1808,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
t.Fatalf("write: %v", err)
}
line := lines.waitFor(t, "set aside an edit of a state file that does not parse")
line := lines.waitFor(t, setAside)
message, _ := line["error"].(string)
if line["file"] != path+".bad" ||
@@ -1637,8 +1823,8 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad")
}
wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON,
reputationJSON)
wantFiles(t, dir, abuseIPDBChecksJSON, abuseIPDBScoresJSON, alertsJSON, bansJSON,
bansJSON+".bad", clientsJSON, dnsblJSON, listsJSON, lookupsJSON)
if got := readFile(t, path+".bad"); got != broken {
t.Errorf("bans.json.bad holds\n%s\nwant the edit", got)
@@ -1766,9 +1952,9 @@ func midnight() time.Time {
return time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
}
// newParams returns Params for the state files in dir, with parts that
// hold nothing yet. GeoJS is never asked, the lists, two blocklists, are
// never fetched, and the alerts, at most two an hour, are
// newParams returns Params for the state and cache files, both in dir,
// with parts that hold nothing yet. GeoJS is never asked, the lists, two
// blocklists, are never fetched, and the alerts, at most two an hour, are
// never sent. The anomaly counters count the scopes fill counts, with
// thresholds fill does not reach.
func newParams(dir string) state.Params {
@@ -1785,6 +1971,7 @@ func newParams(dir string) state.Params {
return state.Params{
Dir: dir,
CacheDir: dir,
WriteDelay: time.Hour,
CounterInterval: time.Hour,
Ledger: bans.New(bans.Params{
@@ -1837,9 +2024,9 @@ func office() netip.Prefix {
// with counts and histories,
// GeoJS answers, the blocklists' last tries and the copy of one, two
// verdicts of a DNSBL zone, and the AbuseIPDB checks spent today with two
// scores, as filledReputationJSON holds them, and alerts
// and anomaly counters, as filledAlertsJSON holds them, into the parts of
// params.
// scores, as filledListsJSON, filledDNSBLJSON, filledAbuseIPDBScoresJSON
// and filledAbuseIPDBChecksJSON hold them, and alerts and anomaly
// counters, as filledAlertsJSON holds them, into the parts of params.
func fill(params state.Params) {
now := midnight()
client := netip.MustParsePrefix("203.0.113.9/32")
@@ -1895,11 +2082,12 @@ func fill(params state.Params) {
},
{Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)},
})
params.AbuseIPDB.Load(reputation.Checks{
Day: civil.DateOf(now), Spent: 3, Scores: []reputation.Score{
{Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)},
{Client: client, Score: 100, Fetched: now.Add(-time.Hour)},
},
params.AbuseIPDB.Load([]reputation.Score{
{Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)},
{Client: client, Score: 100, Fetched: now.Add(-time.Hour)},
})
params.AbuseIPDB.LoadChecksSpent(reputation.ChecksSpent{
Day: civil.DateOf(now), Spent: 3,
})
// An alert waiting, a repeat of it the cooldown holds back, another
@@ -2160,6 +2348,40 @@ func wantRefused(t *testing.T, file, content, want string) {
}
}
// wantSetAside writes content to the cache file named file in a new
// directory, and checks that Load starts all the same, with the cache
// empty, and sets the file aside, logging an error that is the file's
// path and then starts with want.
func wantSetAside(t *testing.T, file, content, want string) {
t.Helper()
dir := t.TempDir()
path := filepath.Join(dir, file)
err := os.WriteFile(path, []byte(content), 0o600)
if err != nil {
t.Fatalf("write %s: %v", file, err)
}
params := newParams(dir)
lines := logInto(&params)
load(t, params)
line := lines.waitFor(t, setAside)
if message, _ := line["error"].(string); line["file"] != path+".bad" ||
!strings.HasPrefix(message, path+want) {
t.Errorf("set aside with %v, want %s.bad and an error starting %s%s", line, path,
path, want)
}
if len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
len(params.DNSBL.Snapshot()) != 0 || len(params.AbuseIPDB.Snapshot()) != 0 {
t.Error("a cache holds what a file set aside held")
}
wantFiles(t, dir, file+".bad")
}
// removeFiles removes the named files from dir.
func removeFiles(t *testing.T, dir string, names ...string) {
t.Helper()