Caches of outside APIs live in /var/cache/sww (closes #314)
check / check (push) Waiting to run

The answers of GeoJS, the lists fetched by URL, the DNS blocklists and
AbuseIPDB each go in a cache file in SWW_CACHE_DIR (default
/var/cache/sww): lookups.json, lists.json, dnsbl.json and
abuseipdb-scores.json. reputation.json is gone. The AbuseIPDB checks
spent today are a count, not a cache, and stay in SWW_STATE_DIR as
abuseipdb-checks.json.

A cache file never stops the start: one that does not parse is set
aside as an edit is, one that cannot be read is written over, and its
cache starts empty. An unwritable SWW_CACHE_DIR stops the start, as an
unwritable SWW_STATE_DIR does. The run script makes the cache directory
and gives it to sww.

Model: opus-5-5
This commit is contained in:
sneak committed 2026-11-11 05:33:55 +00:00
1 parent b89c53c640
commit 7c913d058d
23 files changed
+1049 -449

No files matched your search

+7 -3
View File
@@ -191,12 +191,15 @@ type Config struct {
BanScopeV4Prefix int
// StateDir is the directory of the state files, an absolute path
// (SWW_STATE_DIR). bans.json is written StateWriteDelay after a ban
// is made (SWW_STATE_WRITE_DELAY), and every state file every
// StateCounterInterval (SWW_STATE_COUNTER_INTERVAL). Neither can be
// off.
// is made (SWW_STATE_WRITE_DELAY), and every state and cache file
// every StateCounterInterval (SWW_STATE_COUNTER_INTERVAL). Neither
// can be off.
StateDir string
StateWriteDelay time.Duration
StateCounterInterval time.Duration
// CacheDir is the directory of the cache files, which keep answers
// from outside services, an absolute path (SWW_CACHE_DIR).
CacheDir string
// LogRequestHeaders are the request headers whose values the request
// log gives, in lower case (SWW_LOG_REQUEST_HEADERS).
LogRequestHeaders []string
@@ -560,6 +563,7 @@ func FromEnvironment(
StateDir: env.absolutePath("SWW_STATE_DIR", "/var/lib/sww"),
StateWriteDelay: env.durationNotOff("SWW_STATE_WRITE_DELAY", "10s"),
StateCounterInterval: env.durationNotOff("SWW_STATE_COUNTER_INTERVAL", "15m"),
CacheDir: env.absolutePath("SWW_CACHE_DIR", "/var/cache/sww"),
LogRequestHeaders: env.headerNames("SWW_LOG_REQUEST_HEADERS",
"accept,accept-language,accept-encoding,content-type,origin,range"),
LogLevel: env.logLevel("SWW_LOG_LEVEL", "info"),
+21 -7
View File
@@ -74,6 +74,7 @@ const (
stateDir = "SWW_STATE_DIR"
stateWriteDelay = "SWW_STATE_WRITE_DELAY"
stateCounterInterval = "SWW_STATE_COUNTER_INTERVAL"
cacheDir = "SWW_CACHE_DIR"
adminToken = "SWW_ADMIN_TOKEN" //nolint:gosec // the setting's name
metricsToken = "SWW_METRICS_TOKEN" //nolint:gosec // the setting's name
metricsTopN = "SWW_METRICS_TOP_N"
@@ -262,6 +263,7 @@ func TestDefaults(t *testing.T) {
StateDir: "/var/lib/sww",
StateWriteDelay: 10 * time.Second,
StateCounterInterval: 15 * time.Minute,
CacheDir: "/var/cache/sww",
MetricsToken: "",
MetricsTopN: 50,
RulesDir: "/etc/sww/rules.d",
@@ -346,6 +348,7 @@ func TestValuesAsSet(t *testing.T) {
stateDir: "/srv/waf-state",
stateWriteDelay: "500ms",
stateCounterInterval: "1h",
cacheDir: "/srv/waf-cache",
metricsToken: token,
metricsTopN: "10",
rulesDir: "/srv/waf-rules",
@@ -368,6 +371,7 @@ func TestValuesAsSet(t *testing.T) {
StateDir: "/srv/waf-state",
StateWriteDelay: 500 * time.Millisecond,
StateCounterInterval: time.Hour,
CacheDir: "/srv/waf-cache",
MetricsToken: token,
MetricsTopN: 10,
RulesDir: "/srv/waf-rules",
@@ -2302,6 +2306,7 @@ func TestInvalidBanOrStateValueStopsTheStart(t *testing.T) {
{stateDir, ""}, {stateDir, "state"}, {stateDir, "./var/lib/sww"},
{stateWriteDelay, off}, {stateWriteDelay, "0s"},
{stateCounterInterval, off}, {stateCounterInterval, "15"},
{cacheDir, ""}, {cacheDir, "cache"}, {cacheDir, "./var/cache/sww"},
})
}
@@ -2574,6 +2579,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
stateDir: "/var/lib/sww",
stateWriteDelay: "10s",
stateCounterInterval: "15m",
cacheDir: "/var/cache/sww",
adminToken: "",
metricsToken: "",
metricsTopN: "50",
@@ -2674,8 +2680,8 @@ func wantLookupSettings(t *testing.T, got *config.Config, want config.Config) {
}
}
// wantBanSettings checks the settings for bans, the state files, the
// metrics and the rule files.
// wantBanSettings checks the settings for bans, the state and cache
// files, the metrics and the rule files.
func wantBanSettings(t *testing.T, got *config.Config, want config.Config) {
t.Helper()
@@ -2694,11 +2700,7 @@ func wantBanSettings(t *testing.T, got *config.Config, want config.Config) {
got.RulesDir, got.RulesEnabled, want.RulesDir, want.RulesEnabled)
}
if got.StateDir != want.StateDir ||
got.StateWriteDelay != want.StateWriteDelay ||
got.StateCounterInterval != want.StateCounterInterval {
t.Errorf("state settings\n%+v\nwant\n%+v", got, want)
}
wantStateSettings(t, got, want)
if got.MetricsToken != want.MetricsToken || got.MetricsTopN != want.MetricsTopN {
t.Errorf("metrics token %q and top %d, want %q and %d",
@@ -2706,6 +2708,18 @@ func wantBanSettings(t *testing.T, got *config.Config, want config.Config) {
}
}
// wantStateSettings checks the settings for the state and cache files.
func wantStateSettings(t *testing.T, got *config.Config, want config.Config) {
t.Helper()
if got.StateDir != want.StateDir ||
got.StateWriteDelay != want.StateWriteDelay ||
got.StateCounterInterval != want.StateCounterInterval ||
got.CacheDir != want.CacheDir {
t.Errorf("state settings\n%+v\nwant\n%+v", got, want)
}
}
// wantNetblocks checks a list of netblocks.
func wantNetblocks(t *testing.T, got []netip.Prefix, want ...string) {
t.Helper()
+40 -24
View File
@@ -36,22 +36,21 @@ var (
"checks spent; none is made until the day ends at 00:00 UTC")
)
// Score is what AbuseIPDB said about a client, as reputation.json holds
// it: the client, an IPv4 address or an IPv6 group, its abuse confidence
// score, from 0 to 100, and when AbuseIPDB answered.
// Score is what AbuseIPDB said about a client, as abuseipdb-scores.json
// holds it: the client, an IPv4 address or an IPv6 group, its abuse
// confidence score, from 0 to 100, and when AbuseIPDB answered.
type Score struct {
Client netip.Prefix `json:"client"`
Score int64 `json:"score"`
Fetched time.Time `json:"fetched"`
}
// Checks are what reputation.json keeps of the checks of clients with
// AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the
// first, and the scores still in use.
type Checks struct {
Day civil.Date `json:"day,omitzero"`
Spent int `json:"spent"`
Scores []Score `json:"scores"`
// ChecksSpent is what abuseipdb-checks.json keeps of the checks of
// clients with AbuseIPDB: the day, in UTC, of the checks Spent counts,
// zero before the first.
type ChecksSpent struct {
Day civil.Date
Spent int
}
// AbuseIPDBParams are what NewAbuseIPDB needs.
@@ -196,35 +195,35 @@ func (a *AbuseIPDB) BudgetLeft() int {
return max(a.params.DailyBudget-a.spent, 0)
}
// Snapshot returns the checks spent and every score still in use, sorted
// by client, as reputation.json keeps them.
func (a *AbuseIPDB) Snapshot() Checks {
// Snapshot returns every score still in use, sorted by client, as
// abuseipdb-scores.json keeps them.
func (a *AbuseIPDB) Snapshot() []Score {
a.mu.Lock()
now := a.params.Now()
checks := Checks{Day: a.day, Spent: a.spent, Scores: make([]Score, 0, a.scores.Len())}
scores := make([]Score, 0, a.scores.Len())
for _, kept := range a.scores.Values() {
if now.Sub(kept.Fetched) < a.params.CacheTTL {
checks.Scores = append(checks.Scores, kept)
scores = append(scores, kept)
}
}
a.mu.Unlock()
slices.SortFunc(checks.Scores, func(x, y Score) int {
slices.SortFunc(scores, func(x, y Score) int {
return x.Client.Compare(y.Client)
})
return checks
return scores
}
// Load keeps checks, read from reputation.json, in place of those it
// keeps, but for the scores past maxVerdicts, those fetched longest ago.
// One fetched CacheTTL ago or more is neither used nor written, as for any
// score.
func (a *AbuseIPDB) Load(checks Checks) {
scores := slices.Clone(checks.Scores)
// Load keeps scores, read from abuseipdb-scores.json, in place of those
// it keeps, but for the scores past maxVerdicts, those fetched longest
// ago. One fetched CacheTTL ago or more is neither used nor written, as
// for any score.
func (a *AbuseIPDB) Load(scores []Score) {
scores = slices.Clone(scores)
slices.SortStableFunc(scores, func(x, y Score) int {
return x.Fetched.Compare(y.Fetched)
})
@@ -232,7 +231,6 @@ func (a *AbuseIPDB) Load(checks Checks) {
a.mu.Lock()
defer a.mu.Unlock()
a.day, a.spent = checks.Day, checks.Spent
a.scores.Purge()
for _, kept := range scores {
@@ -240,6 +238,24 @@ func (a *AbuseIPDB) Load(checks Checks) {
}
}
// ChecksSpent returns the checks spent, as abuseipdb-checks.json keeps
// them.
func (a *AbuseIPDB) ChecksSpent() ChecksSpent {
a.mu.Lock()
defer a.mu.Unlock()
return ChecksSpent{Day: a.day, Spent: a.spent}
}
// LoadChecksSpent keeps checks, read from abuseipdb-checks.json, in place
// of the checks spent.
func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
a.mu.Lock()
defer a.mu.Unlock()
a.day, a.spent = checks.Day, checks.Spent
}
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
// the score as client's, and notes the check as no longer under way. A
// check that fails gives no score: it is counted, logged and raised as a
+15 -16
View File
@@ -17,6 +17,7 @@ import (
"testing/synctest"
"time"
"cloud.google.com/go/civil"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -179,11 +180,13 @@ func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) {
t.Errorf("logged\n%s\nwant the budget used up", log.String())
}
// Restarted with what reputation.json keeps, it uses the scores, and
// checks no client until the day ends.
// Restarted with what abuseipdb-scores.json and
// abuseipdb-checks.json keep, it uses the scores, and checks no
// client until the day ends.
restarted := &abuseIPDBStandIn{}
again := newAbuseIPDB(restarted, p)
again.Load(checker.Snapshot())
again.LoadChecksSpent(checker.ChecksSpent())
wantScore(t, again, suspect, true, 100, true)
wantBudgetLeft(t, again, 0)
@@ -224,13 +227,9 @@ func TestChecksAreCountedInUTCDaysWhateverTheClocksZone(t *testing.T) {
time.Sleep(2 * time.Hour)
wantBudgetLeft(t, checker, 0)
encoded, err := json.Marshal(checker.Snapshot())
if err != nil {
t.Fatalf("encode: %v", err)
}
if !strings.Contains(string(encoded), `"day":"2000-01-01"`) {
t.Errorf("reputation.json would hold %s, want the day 2000-01-01", encoded)
want := civil.Date{Year: 2000, Month: 1, Day: 1}
if day := checker.ChecksSpent().Day; day != want {
t.Errorf("checks spent on %s, want on %s", day, want)
}
// At 00:00 UTC the budget is whole again.
@@ -302,7 +301,7 @@ func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) {
wantChecked(t, abuseIPDB, suspect, other)
wantFailures(t, checker, 2)
if scores := checker.Snapshot().Scores; len(scores) != 0 {
if scores := checker.Snapshot(); len(scores) != 0 {
t.Errorf("scores %+v, want none", scores)
}
@@ -399,7 +398,7 @@ func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
for name, shown := range map[string]string{
"the log": log.String(), "the alerts": string(alerted),
"the metrics": scrapeMetrics(t, m), "reputation.json": string(kept),
"the metrics": scrapeMetrics(t, m), "abuseipdb-scores.json": string(kept),
} {
if strings.Contains(shown, key) {
t.Errorf("%s shows the key:\n%s", name, shown)
@@ -462,12 +461,12 @@ func TestScoreFetchedATTLAgoIsNeitherUsedNorKept(t *testing.T) {
Fetched: now.Add(-cacheTTL),
}
checker.Load(reputation.Checks{Scores: []reputation.Score{stale, inUse}})
checker.Load([]reputation.Score{stale, inUse})
wantScore(t, checker, suspect, false, 100, true)
wantScore(t, checker, other, false, 0, false)
got := checker.Snapshot().Scores
got := checker.Snapshot()
if !reflect.DeepEqual(got, []reputation.Score{inUse}) {
t.Errorf("scores %+v, want only %+v", got, inUse)
}
@@ -481,7 +480,7 @@ func TestAtMost100000ScoresKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.T)
p.Now = func() time.Time { return now }
checker := reputation.NewAbuseIPDB(p)
// 100,001 scores, listed by client, as reputation.json lists them, each
// 100,001 scores, listed by client, as abuseipdb-scores.json lists them, each
// fetched a millisecond before the one before it: the last is one too
// many.
const count = 100001
@@ -497,9 +496,9 @@ func TestAtMost100000ScoresKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.T)
addr = addr.Next()
}
checker.Load(reputation.Checks{Scores: scores})
checker.Load(scores)
got := checker.Snapshot().Scores
got := checker.Snapshot()
if len(got) != count-1 || !slices.Contains(got, scores[0]) ||
slices.Contains(got, scores[count-1]) {
t.Errorf("%d scores kept, want all but the one fetched longest ago", len(got))
+2 -2
View File
@@ -291,7 +291,7 @@ func crowdSecFailures() []crowdSecFailure {
// other than its decision list, such as the one a redirect names, and that
// the key is in none of what the fetches leave behind: log, the process
// log, the alerts waiting in queue, and the copies of lists, which
// reputation.json keeps.
// lists.json keeps.
func wantKeyNotShown(
t *testing.T, e *engine, log string, lists *reputation.Lists, queue *alerts.Queue,
) {
@@ -326,7 +326,7 @@ func TestCrowdSecDecisionListKeptAcrossARestartEndsWhenItsDecisionsDo(t *testing
lists := start(t, e, crowdSecParams())
kept := lists.Snapshot()
// Restarted half an hour later with what reputation.json keeps, and
// Restarted half an hour later with what lists.json keeps, and
// the engine down, the decision still bans, until the end it had at
// the fetch, half an hour on.
time.Sleep(30 * time.Minute)
+3 -3
View File
@@ -40,7 +40,7 @@ var (
errNotListing = errors.New("the answer is outside 127.0.0.0/8")
)
// Verdict is what a zone said about a client, as reputation.json holds
// Verdict is what a zone said about a client, as dnsbl.json holds
// it: the zone, the client's address, whether the zone lists it, and when
// the zone answered.
type Verdict struct {
@@ -187,7 +187,7 @@ func (d *DNSBL) Failures(zone string) int {
}
// Snapshot returns every verdict still in use, sorted by client, then by
// zone, as reputation.json lists them.
// zone, as dnsbl.json lists them.
func (d *DNSBL) Snapshot() []Verdict {
d.mu.Lock()
@@ -209,7 +209,7 @@ func (d *DNSBL) Snapshot() []Verdict {
return verdicts
}
// Load keeps verdicts, read from reputation.json, in place of those it
// Load keeps verdicts, read from dnsbl.json, in place of those it
// keeps, but for those of a zone SWW_DNSBL_ZONES does not name, and,
// past maxVerdicts, those fetched longest ago. One fetched CacheTTL ago or
// more is neither used nor written, as for any verdict.
+2 -2
View File
@@ -389,7 +389,7 @@ func TestVerdictsKeptAcrossARestart(t *testing.T) {
t.Errorf("verdicts %+v, want %+v", kept, want)
}
// Restarted an hour later with what reputation.json keeps, it uses
// Restarted an hour later with what dnsbl.json keeps, it uses
// the verdicts, and asks the zone nothing, until the TTL has passed
// since they were fetched.
time.Sleep(time.Hour)
@@ -446,7 +446,7 @@ func TestAtMost100000VerdictsKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.
p.Now = func() time.Time { return now }
dnsbl := reputation.NewDNSBL(p)
// 100,001 verdicts, listed by client, as reputation.json lists them,
// 100,001 verdicts, listed by client, as dnsbl.json lists them,
// each fetched a millisecond before the one before it: the last is one
// too many.
const count = 100001
+7 -7
View File
@@ -7,8 +7,8 @@
// and when each was last tried. It also asks the DNSBL zones of
// SWW_DNSBL_ZONES about clients, and keeps their verdicts, and checks
// clients with AbuseIPDB, and keeps their scores and the checks spent
// today. The state package writes all of these to reputation.json and
// reads them from it, so that a restart keeps them too.
// today. The state package writes all of these to files and reads them
// from there, so that a restart keeps them too.
package reputation
import (
@@ -54,7 +54,7 @@ var (
"does not give an address or a netblock and a duration, such as 4h0m0s")
)
// List is a list as reputation.json holds it: the URL it is fetched from,
// List is a list as lists.json holds it: the URL it is fetched from,
// when it was last tried, the fetch failed or not, and its last good copy:
// when that was fetched, and its lines, as fetched, comment lines
// included, both left out while no fetch of it has succeeded.
@@ -106,7 +106,7 @@ type Lists struct {
lists map[string]*list
}
// list is one list: what reputation.json keeps of it, its last try, zero
// list is one list: what lists.json keeps of it, its last try, zero
// before the first, and its last good copy, what that copy says, and how
// many fetches of it failed.
type list struct {
@@ -272,7 +272,7 @@ func (l *Lists) Failures(listURL string) int {
// Run fetches each list once it is due, as due tells, until ctx is done. A
// list never tried is fetched at once, and so is one that is due by its
// last try or copy read from reputation.json.
// last try or copy read from lists.json.
func (l *Lists) Run(ctx context.Context) {
if len(l.lists) == 0 {
return
@@ -292,7 +292,7 @@ func (l *Lists) Run(ctx context.Context) {
}
// Snapshot returns each list that has been tried, with its copy, if it
// has one, sorted by URL, as reputation.json lists them.
// has one, sorted by URL, as lists.json lists them.
func (l *Lists) Snapshot() []List {
l.mu.Lock()
@@ -313,7 +313,7 @@ func (l *Lists) Snapshot() []List {
return tried
}
// Load puts lists, read from reputation.json, in place of the last tries
// Load puts lists, read from lists.json, in place of the last tries
// and copies held. A list Params does not name is dropped. A copy with a
// line that parse refuses is an error, and then nothing changes.
func (l *Lists) Load(lists []List) error {
+3 -3
View File
@@ -260,7 +260,7 @@ func TestKeptCopyIsFetchedAgainOnceRefreshHasPassedSinceItWasFetched(t *testing.
lists.SetTransport(servers)
// drop.txt was fetched an hour ago, and tor.txt a refresh ago, as
// reputation.json says at start.
// lists.json says at start.
err := lists.Load([]reputation.List{
{URL: dropURL, Fetched: time.Now().Add(-time.Hour), Lines: []string{"203.0.113.9"}},
{URL: torURL, Fetched: time.Now().Add(-refresh), Lines: []string{"198.51.100.9"}},
@@ -297,7 +297,7 @@ func TestRestartWaitsRefreshAfterTheLastTryEvenOneThatFailed(t *testing.T) {
time.Sleep(refresh)
wantFetches(t, servers, 2)
// Restarted with what reputation.json keeps, it waits a refresh
// Restarted with what lists.json keeps, it waits a refresh
// after the failed try, as it does while it runs.
restarted := &standIn{lists: map[string]string{dropURL: "198.51.100.2\n"}}
again := reputation.New(params(dropURL))
@@ -353,7 +353,7 @@ func TestFetchCutOffAsItStopsIsNoFailureButARestartWaitsForIt(t *testing.T) {
waiting(queue))
}
// Restarted an hour later with what reputation.json keeps, it fetches
// Restarted an hour later with what lists.json keeps, it fetches
// tor.txt, never tried, at once, and drop.txt a refresh after its
// cut-off try.
time.Sleep(time.Hour)
+333 -131
View File
@@ -1,13 +1,22 @@
// Package state keeps sww's state in JSON files in
// SWW_STATE_DIR, as the "Persistent state" section of SPEC.md requires
// and docs/state-files.md lists them:
// bans.json holds the bans, clients.json each client's counters and
// history, lookups.json GeoJS's answers, reputation.json the last try and
// last good copy of each list fetched from a URL, the DNSBL zones'
// verdicts, and AbuseIPDB's scores and checks spent, and alerts.json the
// cooldowns, the hour under way, the alerts waiting for each destination
// and the anomaly counters. Load
// reads them at start, Watch takes in an admin's edit of one while
// Package state keeps sww's state in JSON files, as the "Persistent
// state" section of SPEC.md requires and docs/state-files.md lists them.
//
// The state files are in SWW_STATE_DIR:
// - bans.json holds the bans;
// - clients.json holds each client's counters and history;
// - abuseipdb-checks.json holds the AbuseIPDB checks spent today;
// - alerts.json holds the cooldowns, the hour under way, the alerts
// waiting for each destination and the anomaly counters.
//
// The cache files are in SWW_CACHE_DIR. Each keeps answers from an
// outside service, which are fetched again when the file is lost:
// - lookups.json holds GeoJS's answers;
// - lists.json holds the last try and last good copy of each list
// fetched from a URL;
// - dnsbl.json holds the DNSBL zones' verdicts;
// - abuseipdb-scores.json holds AbuseIPDB's scores.
//
// Load reads them at start, Watch takes in an admin's edit of one while
// sww runs, and Run and WriteAll write them. The disk is read and
// written outside the parts' locks, which are held only to take a
// snapshot or to put in what a file holds, so that no request waits on
@@ -31,6 +40,7 @@ import (
"sync"
"time"
"cloud.google.com/go/civil"
"github.com/fsnotify/fsnotify"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
@@ -48,15 +58,41 @@ const version = 1
// hold visitors' addresses.
const fileMode = 0o600
// The state files' names.
// The state files' names, in Dir.
const (
bansJSON = "bans.json"
clientsJSON = "clients.json"
lookupsJSON = "lookups.json"
reputationJSON = "reputation.json"
alertsJSON = "alerts.json"
bansJSON = "bans.json"
clientsJSON = "clients.json"
abuseIPDBChecksJSON = "abuseipdb-checks.json"
alertsJSON = "alerts.json"
)
// The cache files' names, in CacheDir.
const (
lookupsJSON = "lookups.json"
listsJSON = "lists.json"
dnsblJSON = "dnsbl.json"
abuseIPDBScoresJSON = "abuseipdb-scores.json"
)
// fileNames returns the names of the state files, then those of the
// cache files, in the order they are written.
func fileNames() []string {
return []string{
bansJSON, clientsJSON, abuseIPDBChecksJSON, alertsJSON,
lookupsJSON, listsJSON, dnsblJSON, abuseIPDBScoresJSON,
}
}
// isCacheFile reports whether name is the name of a cache file.
func isCacheFile(name string) bool {
switch name {
case lookupsJSON, listsJSON, dnsblJSON, abuseIPDBScoresJSON:
return true
default:
return false
}
}
var (
errVersion = errors.New("unknown version")
// errMissing is for an entry without a field it needs.
@@ -72,8 +108,10 @@ var (
// Params are what Load needs.
type Params struct {
// Dir is the directory of the state files (SWW_STATE_DIR).
Dir string
// Dir is the directory of the state files (SWW_STATE_DIR), and
// CacheDir that of the cache files (SWW_CACHE_DIR).
Dir string
CacheDir string
// WriteDelay is how long after a ban is made bans.json is written
// (SWW_STATE_WRITE_DELAY), and CounterInterval how often every file
// is (SWW_STATE_COUNTER_INTERVAL).
@@ -101,7 +139,7 @@ type Params struct {
Metrics *metrics.Metrics
}
// Files are the state files of a running sww.
// Files are the state and cache files of a running sww.
type Files struct {
params Params
@@ -147,13 +185,32 @@ type lookupsFile struct {
Lookups []lookup.Answer `json:"lookups"`
}
// reputationFile is reputation.json, indented for an admin to read and
// edit, so that each line of a list's copy is on a line of its own.
type reputationFile struct {
Version int `json:"version"`
Lists []reputation.List `json:"lists"`
Verdicts []reputation.Verdict `json:"verdicts"`
AbuseIPDB reputation.Checks `json:"abuseipdb"`
// abuseIPDBChecksFile is abuseipdb-checks.json, indented for an admin to
// read and edit.
type abuseIPDBChecksFile struct {
Version int `json:"version"`
Day civil.Date `json:"day,omitzero"`
Spent int `json:"spent"`
}
// listsFile is lists.json, indented for an admin to read and edit, so
// that each line of a list's copy is on a line of its own.
type listsFile struct {
Version int `json:"version"`
Lists []reputation.List `json:"lists"`
}
// dnsblFile is dnsbl.json, with each verdict on a line of its own.
type dnsblFile struct {
Version int `json:"version"`
Verdicts []reputation.Verdict `json:"verdicts"`
}
// abuseIPDBScoresFile is abuseipdb-scores.json, with each score on a line
// of its own.
type abuseIPDBScoresFile struct {
Version int `json:"version"`
Scores []reputation.Score `json:"scores"`
}
// alertsFile is alerts.json, indented for an admin to read and edit.
@@ -178,34 +235,44 @@ type stateFile interface {
toUTC()
}
// Load checks that files can be written in Dir, and reads the state files
// in it into the parts of Params that hold the state. A missing file is
// empty state, as on a first start. A file that does not parse, has an
// unknown version, or has an entry without a field it needs, is an error
// that names the file and, where the JSON decoder tells it, the line and
// column, or else the entry.
// Load checks that files can be written in Dir and CacheDir, and reads the
// files in them into the parts of Params that hold the state. A missing
// file is empty state, as on a first start. A state file that does not
// parse, has an unknown version, or has an entry without a field it
// needs, is an error that names the file and, where the JSON decoder
// tells it, the line and column, or else the entry. A cache file is never
// an error, as readCache describes.
func Load(params Params) (*Files, error) {
err := checkWritable(params.Dir)
if err != nil {
return nil, fmt.Errorf("SWW_STATE_DIR cannot be written: %w", err)
}
err = checkWritable(params.CacheDir)
if err != nil {
return nil, fmt.Errorf("SWW_CACHE_DIR cannot be written: %w", err)
}
f := &Files{params: params, sums: map[string][sha256.Size]byte{}}
bansRead, bansErr := f.read(bansJSON)
clientsRead, clientsErr := f.read(clientsJSON)
lookupsRead, lookupsErr := f.read(lookupsJSON)
reputationRead, reputationErr := f.read(reputationJSON)
_, checksErr := f.read(abuseIPDBChecksJSON)
alertsRead, alertsErr := f.read(alertsJSON)
err = errors.Join(bansErr, clientsErr, lookupsErr, reputationErr, alertsErr)
err = errors.Join(bansErr, clientsErr, checksErr, alertsErr)
if err != nil {
return nil, err
}
lookupsRead := f.readCache(lookupsJSON)
listsRead := f.readCache(listsJSON)
f.readCache(dnsblJSON)
f.readCache(abuseIPDBScoresJSON)
params.ProcessLog.Info("read the state files", "directory", params.Dir,
"bans", bansRead, "clients", clientsRead, "lookups", lookupsRead,
"lists", reputationRead, "alerts_waiting", alertsRead)
"cache_directory", params.CacheDir, "bans", bansRead, "clients", clientsRead,
"lookups", lookupsRead, "lists", listsRead, "alerts_waiting", alertsRead)
return f, nil
}
@@ -234,28 +301,33 @@ func (f *Files) Run(ctx context.Context) {
f.logFailure(bansJSON, f.writeFile(bansJSON))
case <-interval.C:
for _, name := range []string{
bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
} {
for _, name := range fileNames() {
f.logFailure(name, f.writeFile(name))
}
}
}
}
// WriteAll writes every state file, as sww stops. A file that
// WriteAll writes every state and cache file, as sww stops. A file that
// fails does not keep the others from being written.
func (f *Files) WriteAll() error {
return errors.Join(f.writeFile(bansJSON), f.writeFile(clientsJSON),
f.writeFile(lookupsJSON), f.writeFile(reputationJSON), f.writeFile(alertsJSON))
names := fileNames()
errs := make([]error, 0, len(names))
for _, name := range names {
errs = append(errs, f.writeFile(name))
}
return errors.Join(errs...)
}
// Watch watches Dir until ctx is done, and takes in an admin's edit of a
// state file as soon as it is saved: what the file holds replaces what
// sww held for it. An edit that does not parse is left for the
// file's next write, which sets it aside, since a file can be read while
// an editor is still writing it. If Dir cannot be watched, that is
// logged, and an edit is taken in only before its file is written.
// Watch watches Dir and CacheDir until ctx is done, and takes in an
// admin's edit of a file as soon as it is saved: what the file holds
// replaces what sww held for it. An edit that does not parse is left for
// the file's next write, which sets it aside, since a file can be read
// while an editor is still writing it. If a directory cannot be watched,
// that is logged, and an edit is taken in only before its file is
// written.
func (f *Files) Watch(ctx context.Context) {
watcher, err := fsnotify.NewWatcher()
if err == nil {
@@ -263,7 +335,7 @@ func (f *Files) Watch(ctx context.Context) {
_ = watcher.Close()
}()
err = watcher.Add(f.params.Dir)
err = errors.Join(watcher.Add(f.params.Dir), watcher.Add(f.params.CacheDir))
}
if err != nil {
@@ -274,15 +346,16 @@ func (f *Files) Watch(ctx context.Context) {
}
f.params.ProcessLog.Info("watching the state files for edits",
"directory", f.params.Dir)
"directory", f.params.Dir, "cache_directory", f.params.CacheDir)
for {
select {
case <-ctx.Done():
return
case event := <-watcher.Events:
switch name := filepath.Base(event.Name); name {
case bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON:
name := filepath.Base(event.Name)
if slices.Contains(fileNames(), name) &&
filepath.Join(f.dir(name), name) == event.Name {
f.fileChanged(name)
}
case err = <-watcher.Errors:
@@ -292,7 +365,7 @@ func (f *Files) Watch(ctx context.Context) {
}
}
// logFailure logs a write of the state file name that failed, and raises
// logFailure logs a write of the file name that failed, and raises
// a file_error alert for it.
func (f *Files) logFailure(name string, err error) {
if err != nil {
@@ -304,14 +377,14 @@ func (f *Files) logFailure(name string, err error) {
Event: alerts.EventFileError,
Reason: failed,
Detail: map[string]any{
"file": filepath.Join(f.params.Dir, name), "error": err.Error(),
"file": filepath.Join(f.dir(name), name), "error": err.Error(),
},
})
f.params.ProcessLog.Error(failed, "error", err.Error())
}
}
// fileChanged takes in what the state file name holds, as Watch sees it
// fileChanged takes in what the file name holds, as Watch sees it
// change, if that is an edit made since sww last read or wrote
// the file. A file that cannot be read or does not parse is left for its
// next write.
@@ -327,7 +400,7 @@ func (f *Files) fileChanged(name string) {
_ = f.takeInEdit(name, data)
}
// takeInEdit takes in data, an edit of the state file name, as takeIn
// takeInEdit takes in data, an edit of the file name, as takeIn
// does, and counts and logs it. Every edit taken in while sww
// runs, by Watch or by a write, is taken in here. An edit that does not
// parse is neither counted nor logged, and takeIn's error returned.
@@ -341,13 +414,23 @@ func (f *Files) takeInEdit(name string, data []byte) error {
// log line is.
f.params.Metrics.StateFileEditTakenIn(name)
f.params.ProcessLog.Info("took in an edit of a state file",
"file", filepath.Join(f.params.Dir, name))
"file", filepath.Join(f.dir(name), name))
return nil
}
// read takes in the state file name at start, and returns how many
// entries it holds. A missing file holds none.
// dir returns the directory of the file name: CacheDir for a cache file,
// and Dir for a state file.
func (f *Files) dir(name string) string {
if isCacheFile(name) {
return f.params.CacheDir
}
return f.params.Dir
}
// read takes in the file name at start, and returns how many entries it
// holds. A missing file holds none.
func (f *Files) read(name string) (int, error) {
data, changed, err := f.readChanged(name)
if err != nil || !changed {
@@ -357,14 +440,44 @@ func (f *Files) read(name string) (int, error) {
return f.takeIn(name, data, false)
}
// readChanged returns what the state file name holds, and whether that
// has changed since sww last read or wrote the file, as it has
// for a file sww never read or wrote. A missing file has not
// changed: it is written again at its next write.
func (f *Files) readChanged(name string) ([]byte, bool, error) {
path := filepath.Join(f.params.Dir, name)
// readCache takes in the cache file name at start, as read does. A cache
// file that cannot be read, or does not parse, does not stop the start:
// it is logged, and its cache starts empty and fills again. One that does
// not parse is set aside as an edit is, and one that cannot be read is
// written over at its next write.
func (f *Files) readCache(name string) int {
data, changed, err := f.readChanged(name)
if err != nil || !changed {
f.logUnread(name, err)
data, err := os.ReadFile(path) //nolint:gosec // a state file, in SWW_STATE_DIR
return 0
}
entries, err := f.takeIn(name, data, false)
if err != nil {
f.logUnread(name, f.setAside(name, err))
}
return entries
}
// logUnread logs err, the reason the cache file name was not read at
// start, unless it is nil.
func (f *Files) logUnread(name string, err error) {
if err != nil {
f.params.ProcessLog.Warn("cannot read a cache file, so its cache starts empty",
"file", filepath.Join(f.dir(name), name), "error", err.Error())
}
}
// readChanged returns what the file name holds, and whether that has
// changed since sww last read or wrote the file, as it has for a file
// sww never read or wrote. A missing file has not changed: it is written
// again at its next write.
func (f *Files) readChanged(name string) ([]byte, bool, error) {
path := filepath.Join(f.dir(name), name)
data, err := os.ReadFile(path) //nolint:gosec // a file in one of the two directories
if errors.Is(err, fs.ErrNotExist) {
return nil, false, nil
}
@@ -376,15 +489,39 @@ func (f *Files) readChanged(name string) ([]byte, bool, error) {
return data, sha256.Sum256(data) != f.sums[name], nil
}
// takeIn parses data, what the state file name holds, puts it into the
// takeIn parses data, what the file name holds, puts it into the
// part that keeps that state, in place of what the part held, and returns
// how many entries the file holds. edit is whether data is an admin's
// edit taken in while sww runs, rather than the file read at the
// start. An error names the file and, where the JSON decoder tells it,
// the line and column, or else the entry.
// the line and column, or else the entry. abuseipdb-checks.json holds
// no entries.
func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
path := filepath.Join(f.params.Dir, name)
path := filepath.Join(f.dir(name), name)
var (
entries int
err error
)
if isCacheFile(name) {
entries, err = f.takeInCache(name, path, data)
} else {
entries, err = f.takeInState(name, path, data, edit)
}
if err != nil {
return 0, err
}
f.sums[name] = sha256.Sum256(data)
return entries, nil
}
// takeInState parses data, what the state file name, at path, holds, and
// puts it in, as takeIn describes.
func (f *Files) takeInState(name, path string, data []byte, edit bool) (int, error) {
var entries int
switch name {
@@ -418,6 +555,33 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
f.params.Limiter.Load(file.Clients, f.params.Now())
entries = len(file.Clients)
case abuseIPDBChecksJSON:
var file abuseIPDBChecksFile
err := parse(path, data, &file)
if err != nil {
return 0, err
}
f.params.AbuseIPDB.LoadChecksSpent(reputation.ChecksSpent{
Day: file.Day, Spent: file.Spent,
})
case alertsJSON:
waiting, err := f.takeInAlerts(path, data)
if err != nil {
return 0, err
}
entries = waiting
}
return entries, nil
}
// takeInCache parses data, what the cache file name, at path, holds, and
// puts it in, as takeIn describes.
func (f *Files) takeInCache(name, path string, data []byte) (int, error) {
switch name {
case lookupsJSON:
var file lookupsFile
@@ -427,9 +591,10 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
}
f.params.GeoJS.Load(file.Lookups)
entries = len(file.Lookups)
case reputationJSON:
var file reputationFile
return len(file.Lookups), nil
case listsJSON:
var file listsFile
err := parse(path, data, &file)
if err != nil {
@@ -441,21 +606,30 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
return 0, fmt.Errorf("%s: %w", path, err)
}
f.params.DNSBL.Load(file.Verdicts)
f.params.AbuseIPDB.Load(file.AbuseIPDB)
entries = len(file.Lists)
case alertsJSON:
waiting, err := f.takeInAlerts(path, data)
return len(file.Lists), nil
case dnsblJSON:
var file dnsblFile
err := parse(path, data, &file)
if err != nil {
return 0, err
}
entries = waiting
f.params.DNSBL.Load(file.Verdicts)
return len(file.Verdicts), nil
default: // abuseipdb-scores.json
var file abuseIPDBScoresFile
err := parse(path, data, &file)
if err != nil {
return 0, err
}
f.params.AbuseIPDB.Load(file.Scores)
return len(file.Scores), nil
}
f.sums[name] = sha256.Sum256(data)
return entries, nil
}
// takeInAlerts parses data, what alerts.json, at path, holds, puts it
@@ -493,17 +667,22 @@ func (f *Files) takeInAlerts(path string, data []byte) (int, error) {
return entries, nil
}
// writeFile writes the state file name from what sww holds. An
// edit made since sww last read or wrote the file is taken in
// first, so that it is not overwritten, or set aside if it does not
// parse. A file that cannot be read, or an edit that cannot be set
// aside, is left as it is, and the write given up. Every write is counted
// in the metrics, and one that fails or is given up as a failure.
// writeFile writes the file name from what sww holds. An edit made since
// sww last read or wrote the file is taken in first, so that it is not
// overwritten, or set aside if it does not parse. A state file that
// cannot be read, or an edit that cannot be set aside, is left as it is,
// and the write given up. A cache file that cannot be read is written
// over, since what it held is fetched again. Every write is counted in
// the metrics, and one that fails or is given up as a failure.
func (f *Files) writeFile(name string) error {
f.mu.Lock()
defer f.mu.Unlock()
data, changed, err := f.readChanged(name)
if err != nil && isCacheFile(name) {
err = nil
}
if err == nil && changed {
err = f.takeInEdit(name, data)
if err != nil {
@@ -519,7 +698,7 @@ func (f *Files) writeFile(name string) error {
}
if err == nil {
err = write(f.params.Dir, name, data)
err = write(f.dir(name), name, data)
}
if err == nil {
@@ -527,7 +706,7 @@ func (f *Files) writeFile(name string) error {
// fails, so that its next read does not take it for an admin's
// edit.
f.sums[name] = sha256.Sum256(data)
err = syncDirectory(f.params.Dir)
err = syncDirectory(f.dir(name))
}
f.params.Metrics.StateFileWritten(name, len(data), err)
@@ -535,13 +714,13 @@ func (f *Files) writeFile(name string) error {
return err
}
// setAside renames the state file name, an edit that does not parse with
// setAside renames the file name, an edit that does not parse with
// parseErr, to name.bad, for the admin to mend, and logs it with where in
// the file the error is, and raises a file_error alert for it. If the
// rename fails, the edit is left as it is, and the error returned is
// parseErr joined with the rename's.
func (f *Files) setAside(name string, parseErr error) error {
path := filepath.Join(f.params.Dir, name)
path := filepath.Join(f.dir(name), name)
err := os.Rename(path, path+".bad")
if err != nil {
@@ -563,7 +742,7 @@ func (f *Files) setAside(name string, parseErr error) error {
return nil
}
// encode returns the state file name as sww writes it, from a
// encode returns the file name as sww writes it, from a
// snapshot of the part that keeps that state.
func (f *Files) encode(name string) ([]byte, error) {
switch name {
@@ -575,10 +754,17 @@ func (f *Files) encode(name string) ([]byte, error) {
return encodeOnePerLine("clients", f.params.Limiter.Snapshot())
case lookupsJSON:
return encodeOnePerLine("lookups", f.params.GeoJS.Snapshot())
case reputationJSON:
return encodeIndented(reputationFile{
Version: version, Lists: f.params.Lists.Snapshot(),
Verdicts: f.params.DNSBL.Snapshot(), AbuseIPDB: f.params.AbuseIPDB.Snapshot(),
case listsJSON:
return encodeIndented(listsFile{Version: version, Lists: f.params.Lists.Snapshot()})
case dnsblJSON:
return encodeOnePerLine("verdicts", f.params.DNSBL.Snapshot())
case abuseIPDBScoresJSON:
return encodeOnePerLine("scores", f.params.AbuseIPDB.Snapshot())
case abuseIPDBChecksJSON:
checks := f.params.AbuseIPDB.ChecksSpent()
return encodeIndented(abuseIPDBChecksFile{
Version: version, Day: checks.Day, Spent: checks.Spent,
})
default: // alerts.json
held := f.params.Alerts.Snapshot()
@@ -748,14 +934,8 @@ func (f *lookupsFile) check(data []byte) error {
// check refuses a list without its URL, which would name no list, or the
// time it was last tried, which would have it fetched at once, and a copy
// of it without the time it was fetched, or without its lines, which hold
// the list. It refuses a verdict without its zone or its client, which
// would be about no one, whether the zone lists the client, or the time
// it was fetched, which would drop it, and so an AbuseIPDB score without
// its client, the score, or the time it was fetched. A verdict's listed is
// false for a client the zone does not list, and a score can be 0, which
// the structs cannot tell from a missing one, so each is read again as
// written.
func (f *reputationFile) check(data []byte) error {
// the list.
func (f *listsFile) check([]byte) error {
for i, kept := range f.Lists {
switch {
case kept.URL == "":
@@ -769,6 +949,15 @@ func (f *reputationFile) check(data []byte) error {
}
}
return nil
}
// check refuses a verdict without its zone or its client, which would be
// about no one, whether the zone lists the client, or the time it was
// fetched, which would drop it. A verdict's listed is false for a client
// the zone does not list, which Verdict cannot tell from a missing one,
// so each listed is read again as written.
func (f *dnsblFile) check(data []byte) error {
var written struct {
Verdicts []struct {
Listed *bool `json:"listed"`
@@ -783,28 +972,27 @@ func (f *reputationFile) check(data []byte) error {
for i, verdict := range f.Verdicts {
switch {
case verdict.Zone == "":
return fmt.Errorf("verdicts %w", missing(i, "zone"))
return missing(i, "zone")
case !verdict.Client.IsValid():
return fmt.Errorf("verdicts %w", missing(i, "client"))
return missing(i, "client")
case written.Verdicts[i].Listed == nil:
return fmt.Errorf("verdicts %w", missing(i, "listed"))
return missing(i, "listed")
case verdict.Fetched.IsZero():
return fmt.Errorf("verdicts %w", missing(i, "fetched"))
return missing(i, "fetched")
}
}
return checkScores(f.AbuseIPDB.Scores, data)
return nil
}
// checkScores refuses an AbuseIPDB score, of scores, read from data, as
// reputationFile's check describes.
func checkScores(scores []reputation.Score, data []byte) error {
// check refuses an AbuseIPDB score without its client, the score, or the
// time it was fetched. A score can be 0, which Score cannot tell from a
// missing one, so each score is read again as written.
func (f *abuseIPDBScoresFile) check(data []byte) error {
var written struct {
AbuseIPDB struct {
Scores []struct {
Score *int64 `json:"score"`
} `json:"scores"`
} `json:"abuseipdb"`
Scores []struct {
Score *int64 `json:"score"`
} `json:"scores"`
}
err := json.Unmarshal(data, &written)
@@ -812,20 +1000,26 @@ func checkScores(scores []reputation.Score, data []byte) error {
return err
}
for i, kept := range scores {
for i, kept := range f.Scores {
switch {
case !kept.Client.IsValid():
return fmt.Errorf("abuseipdb scores %w", missing(i, "client"))
case written.AbuseIPDB.Scores[i].Score == nil:
return fmt.Errorf("abuseipdb scores %w", missing(i, "score"))
return missing(i, "client")
case written.Scores[i].Score == nil:
return missing(i, "score")
case kept.Fetched.IsZero():
return fmt.Errorf("abuseipdb scores %w", missing(i, "fetched"))
return missing(i, "fetched")
}
}
return nil
}
// check refuses nothing, since a missing day or count means no check
// spent.
func (f *abuseIPDBChecksFile) check([]byte) error {
return nil
}
// check refuses a cooldown without its event or when its alert was sent,
// which would hold back no repeat, alerts waiting for a destination with
// another name than webhook, slack or ntfy, most likely misspelt, an
@@ -947,22 +1141,30 @@ func (f *lookupsFile) toUTC() {
}
}
// toUTC converts every time of each list, verdict and AbuseIPDB score to
// UTC.
func (f *reputationFile) toUTC() {
// toUTC converts both times of each list to UTC.
func (f *listsFile) toUTC() {
for i := range f.Lists {
convertToUTC(&f.Lists[i].Tried, &f.Lists[i].Fetched)
}
}
// toUTC converts the time of each verdict to UTC.
func (f *dnsblFile) toUTC() {
for i := range f.Verdicts {
convertToUTC(&f.Verdicts[i].Fetched)
}
}
for i := range f.AbuseIPDB.Scores {
convertToUTC(&f.AbuseIPDB.Scores[i].Fetched)
// toUTC converts the time of each AbuseIPDB score to UTC.
func (f *abuseIPDBScoresFile) toUTC() {
for i := range f.Scores {
convertToUTC(&f.Scores[i].Fetched)
}
}
// toUTC does nothing: abuseipdb-checks.json holds a day, not a time.
func (f *abuseIPDBChecksFile) toUTC() {}
// toUTC converts every time of each cooldown, of the hour, of each alert
// waiting and of each anomaly counter to UTC.
func (f *alertsFile) toUTC() {
@@ -1114,7 +1316,7 @@ func write(dir, name string, data []byte) error {
// syncDirectory syncs dir to the disk, so that a rename in it lasts.
func syncDirectory(dir string) error {
directory, err := os.Open(dir) //nolint:gosec // SWW_STATE_DIR itself
directory, err := os.Open(dir) //nolint:gosec // SWW_STATE_DIR or SWW_CACHE_DIR itself
if err != nil {
return err
}
@@ -1124,7 +1326,7 @@ func syncDirectory(dir string) error {
// writeSynced writes data to the file at path, and syncs it to the disk.
func writeSynced(path string, data []byte) error {
//nolint:gosec // a state file's temporary file, in SWW_STATE_DIR
//nolint:gosec // a file's temporary file, in SWW_STATE_DIR or SWW_CACHE_DIR
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileMode)
if err != nil {
return err
+371 -149
View File
@@ -35,11 +35,21 @@ import (
const (
// The state files.
bansJSON = "bans.json"
clientsJSON = "clients.json"
lookupsJSON = "lookups.json"
reputationJSON = "reputation.json"
alertsJSON = "alerts.json"
bansJSON = "bans.json"
clientsJSON = "clients.json"
abuseIPDBChecksJSON = "abuseipdb-checks.json"
alertsJSON = "alerts.json"
// The cache files.
lookupsJSON = "lookups.json"
listsJSON = "lists.json"
dnsblJSON = "dnsbl.json"
abuseIPDBScoresJSON = "abuseipdb-scores.json"
// What the process log says as a file that does not parse is set aside.
setAside = "set aside an edit of a state file that does not parse"
// What Load says, after a file's path, of a first entry without its
// client or the time it was fetched.
noClient = `: entry 1 has no "client"`
noFetched = `: entry 1 has no "fetched"`
// blocklistURL and torURL are the blocklists the tests' lists name, and
// dnsblZone the DNSBL zone of the tests' verdicts.
blocklistURL = "https://lists.example/drop.txt"
@@ -223,11 +233,9 @@ const filledAlertsJSON = `{
}
`
// filledReputationJSON is reputation.json holding the blocklists' last
// tries and the copy of one, with its comment line, two verdicts of a
// DNSBL zone, and the AbuseIPDB checks spent today with two scores, as
// fill puts them in.
const filledReputationJSON = `{
// filledListsJSON is lists.json holding the blocklists' last tries and
// the copy of one, with its comment line, as fill puts them in.
const filledListsJSON = `{
"version": 1,
"lists": [
{
@@ -244,40 +252,52 @@ const filledReputationJSON = `{
"url": "https://lists.example/tor.txt",
"tried": "2026-10-06T00:00:00Z"
}
],
"verdicts": [
{
"zone": "dnsbl.example",
"client": "203.0.113.9",
"listed": true,
"fetched": "2026-10-05T23:00:00Z"
},
{
"zone": "dnsbl.example",
"client": "2001:db8::1",
"listed": false,
"fetched": "2026-10-05T22:00:00Z"
}
],
"abuseipdb": {
"day": "2026-10-06",
"spent": 3,
"scores": [
{
"client": "203.0.113.9/32",
"score": 100,
"fetched": "2026-10-05T23:00:00Z"
},
{
"client": "2001:db8::/64",
"score": 0,
"fetched": "2026-10-05T22:00:00Z"
}
]
}
]
}
`
// filledDNSBLJSON is dnsbl.json holding the two verdicts of a DNSBL zone
// fill puts in.
const filledDNSBLJSON = `{
"version": 1,
"verdicts": [
{"zone":"dnsbl.example","client":"203.0.113.9","listed":true,` +
`"fetched":"2026-10-05T23:00:00Z"},
{"zone":"dnsbl.example","client":"2001:db8::1","listed":false,` +
`"fetched":"2026-10-05T22:00:00Z"}
]
}
`
// filledAbuseIPDBScoresJSON is abuseipdb-scores.json holding the two
// AbuseIPDB scores fill puts in.
const filledAbuseIPDBScoresJSON = `{
"version": 1,
"scores": [
{"client":"203.0.113.9/32","score":100,"fetched":"2026-10-05T23:00:00Z"},
{"client":"2001:db8::/64","score":0,"fetched":"2026-10-05T22:00:00Z"}
]
}
`
// filledAbuseIPDBChecksJSON is abuseipdb-checks.json holding the
// AbuseIPDB checks spent today, as fill puts them in.
const filledAbuseIPDBChecksJSON = `{
"version": 1,
"day": "2026-10-06",
"spent": 3
}
`
// everyFile returns the names of the state and cache files, sorted, as
// wantFiles lists them in the one directory newParams gives both.
func everyFile() []string {
return []string{
abuseIPDBChecksJSON, abuseIPDBScoresJSON, alertsJSON, bansJSON, clientsJSON,
dnsblJSON, listsJSON, lookupsJSON,
}
}
func TestFilesWrittenAndReadBack(t *testing.T) {
t.Parallel()
@@ -306,14 +326,16 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
if got, want := after.Lists.Snapshot(), before.Lists.Snapshot(); !reflect.DeepEqual(
got, want) {
t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, got, want)
t.Errorf("%s read back\n%+v\nwant\n%+v", listsJSON, got, want)
}
wantEqual(t, reputationJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot())
wantEqual(t, dnsblJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot())
wantEqual(t, abuseIPDBScoresJSON, after.AbuseIPDB.Snapshot(),
before.AbuseIPDB.Snapshot())
checks, wantChecks := after.AbuseIPDB.Snapshot(), before.AbuseIPDB.Snapshot()
if !reflect.DeepEqual(checks, wantChecks) {
t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, checks, wantChecks)
checks, wantChecks := after.AbuseIPDB.ChecksSpent(), before.AbuseIPDB.ChecksSpent()
if checks != wantChecks {
t.Errorf("%s read back %+v, want %+v", abuseIPDBChecksJSON, checks, wantChecks)
}
if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
@@ -324,15 +346,45 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
wantEqual(t, alertsJSON, after.Anomalies.Snapshot(), before.Anomalies.Snapshot())
// Each one-per-line file lists its entries by client, and nothing
// but the five files is left in the directory.
// but the eight files is left in the directory.
wantEntries(t, filepath.Join(dir, clientsJSON), "clients",
"192.0.2.1/32", "203.0.113.9/32", "2001:db8::/64")
wantEntries(t, filepath.Join(dir, lookupsJSON), "lookups",
"192.0.2.1/32", "203.0.113.9/32")
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
wantFiles(t, dir, everyFile()...)
}
func TestReputationJSONKeepsEachCopyWholeOneLineToALine(t *testing.T) {
func TestStateFilesInDirAndCacheFilesInCacheDir(t *testing.T) {
t.Parallel()
stateDir, cacheDir := t.TempDir(), t.TempDir()
before := newParams(stateDir)
before.CacheDir = cacheDir
fill(before)
err := load(t, before).WriteAll()
if err != nil {
t.Fatalf("write: %v", err)
}
wantFiles(t, stateDir, abuseIPDBChecksJSON, alertsJSON, bansJSON, clientsJSON)
wantFiles(t, cacheDir, abuseIPDBScoresJSON, dnsblJSON, listsJSON, lookupsJSON)
// Read back from both, as at the next start.
after := newParams(stateDir)
after.CacheDir = cacheDir
load(t, after)
wantEqual(t, bansJSON, after.Ledger.Snapshot(), before.Ledger.Snapshot())
wantEqual(t, lookupsJSON, after.GeoJS.Snapshot(), before.GeoJS.Snapshot())
checks, wantChecks := after.AbuseIPDB.ChecksSpent(), before.AbuseIPDB.ChecksSpent()
if checks != wantChecks {
t.Errorf("%s read back %+v, want %+v", abuseIPDBChecksJSON, checks, wantChecks)
}
}
func TestReputationFilesAsWritten(t *testing.T) {
t.Parallel()
dir := t.TempDir()
@@ -346,9 +398,16 @@ func TestReputationJSONKeepsEachCopyWholeOneLineToALine(t *testing.T) {
t.Fatalf("write: %v", err)
}
got := readFile(t, filepath.Join(dir, reputationJSON))
if got != filledReputationJSON {
t.Errorf("reputation.json\n%s\nwant\n%s", got, filledReputationJSON)
// lists.json keeps each copy whole, one line to a line.
for name, want := range map[string]string{
listsJSON: filledListsJSON,
dnsblJSON: filledDNSBLJSON,
abuseIPDBScoresJSON: filledAbuseIPDBScoresJSON,
abuseIPDBChecksJSON: filledAbuseIPDBChecksJSON,
} {
if got := readFile(t, filepath.Join(dir, name)); got != want {
t.Errorf("%s\n%s\nwant\n%s", name, got, want)
}
}
}
@@ -473,11 +532,11 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
load(t, params)
held := params.Alerts.Snapshot()
checks := params.AbuseIPDB.Snapshot()
if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
len(params.DNSBL.Snapshot()) != 0 || len(checks.Scores) != 0 || checks.Spent != 0 ||
len(params.DNSBL.Snapshot()) != 0 || len(params.AbuseIPDB.Snapshot()) != 0 ||
params.AbuseIPDB.ChecksSpent().Spent != 0 ||
len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 ||
held.Hour.Sent != 0 {
t.Error("state from no files")
@@ -507,14 +566,14 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
},
{
// Found at the newline that ends the file.
"a cut-off file", lookupsJSON,
"{\n \"version\": 1,\n \"lookups\": [\n",
", line 3, column 17: unexpected end of JSON input",
"a cut-off file", alertsJSON,
"{\n \"version\": 1,\n \"cooldowns\": [\n",
", line 3, column 19: unexpected end of JSON input",
},
{
"an unknown field", lookupsJSON,
`{"version": 1, "lookups": [{"client": "203.0.113.9/32", "contry": "DE"}]}`,
`: json: unknown field "contry"`,
"an unknown field", abuseIPDBChecksJSON,
`{"version": 1, "day": "2026-10-06", "spnet": 3}`,
`: json: unknown field "spnet"`,
},
{
"a netblock that does not read", bansJSON,
@@ -538,14 +597,6 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
`: anomaly_counters entry 2's scope "nett" is not client, net, asn, total ` +
`or watched_net`,
},
{
"a copy of a list with a line that does not read", reputationJSON,
`{"version": 1, "lists": [{"url": "` + blocklistURL + `", ` +
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", ` +
`"lines": ["; DROP", "203.0.113.300"]}]}`,
`: the copy of ` + blocklistURL + `: line 2 is not an address or a netblock, ` +
`such as 192.0.2.0/24`,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
@@ -555,13 +606,91 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
}
}
func TestCacheFileThatDoesNotParseIsSetAsideAndTheCacheStartsEmpty(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name, file, content string
// want is what the error says after the file's path.
want string
}{
{
// Found at the newline that ends the file.
"a cut-off file", lookupsJSON,
"{\n \"version\": 1,\n \"lookups\": [\n",
", line 3, column 17: unexpected end of JSON input",
},
{
"an unknown field", lookupsJSON,
`{"version": 1, "lookups": [{"client": "203.0.113.9/32", "contry": "DE"}]}`,
`: json: unknown field "contry"`,
},
{
"a copy of a list with a line that does not read", listsJSON,
`{"version": 1, "lists": [{"url": "` + blocklistURL + `", ` +
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", ` +
`"lines": ["; DROP", "203.0.113.300"]}]}`,
`: the copy of ` + blocklistURL + `: line 2 is not an address or a netblock, ` +
`such as 192.0.2.0/24`,
},
{
"a verdict of the wrong kind", dnsblJSON,
`{"version": 1, "verdicts": [{"zone": "` + dnsblZone + `", "listed": "yes"}]}`,
`, line 1, column `,
},
{
"an unknown version", abuseIPDBScoresJSON, `{"version": 2}`,
": unknown version ",
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantSetAside(t, tc.file, tc.content, tc.want)
})
}
}
func TestCacheFileThatCannotBeReadIsWrittenOver(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, lookupsJSON)
// lookups.json is a socket, which cannot be opened as a file, even by
// root, but which a rename can replace.
socket, err := (&net.ListenConfig{}).Listen(t.Context(), "unix", path)
if err != nil {
t.Fatalf("listen: %v", err)
}
defer func() {
_ = socket.Close()
}()
params := newParams(dir)
lines := logInto(&params)
files := load(t, params)
line := lines.waitFor(t, "cannot read a cache file, so its cache starts empty")
if line["file"] != path {
t.Errorf("logged %v, want the file %s", line, path)
}
err = files.WriteAll()
if err != nil {
t.Fatalf("write: %v", err)
}
wantEntries(t, path, "lookups")
}
func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
t.Parallel()
const (
// The other fields each entry needs.
ban = `"start": "2026-10-06T00:00:00Z", "expires": null`
answer = `"answered": "2026-10-06T00:00:00Z"`
// The other fields each ban needs.
ban = `"start": "2026-10-06T00:00:00Z", "expires": null`
noNetblock = `: entry 1 has no "netblock"`
)
@@ -601,7 +730,7 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
{
"a client without its address", clientsJSON,
`{"version": 1, "clients": [{"history": {"requests": 3}}]}`,
`: entry 1 has no "client"`,
noClient,
},
{
"a client with requests in a window without its start", clientsJSON,
@@ -615,20 +744,40 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
`"minute_bytes": {"previous": 5120}}]}`,
`: entry 1 has no "minute_bytes.start"`,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantRefused(t, tc.file, tc.content, tc.want)
})
}
}
func TestLookupsJSONEntryWithoutAFieldItNeedsIsSetAside(t *testing.T) {
t.Parallel()
// answer is the other field each answer needs.
const answer = `"answered": "2026-10-06T00:00:00Z"`
for _, tc := range []struct {
name, content string
// want is what the error says after the file's path.
want string
}{
{
"an answer without a client", lookupsJSON,
"an answer without a client",
`{"version": 1, "lookups": [{"country": "DE", ` + answer + `}]}`,
`: entry 1 has no "client"`,
noClient,
},
{
// A country of "" is a client GeoJS cannot place.
"an answer without a country", lookupsJSON,
"an answer without a country",
`{"version": 1, "lookups": [{"client": "192.0.2.1/32", "country": "", ` +
answer + `}, {"client": "203.0.113.9/32", ` + answer + `}]}`,
`: entry 2 has no "country"`,
},
{
"an answer without the time GeoJS gave it", lookupsJSON,
"an answer without the time GeoJS gave it",
`{"version": 1, "lookups": [{"client": "203.0.113.9/32", "country": "DE"}]}`,
`: entry 1 has no "answered"`,
},
@@ -636,7 +785,7 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantRefused(t, tc.file, tc.content, tc.want)
wantSetAside(t, lookupsJSON, tc.content, tc.want)
})
}
}
@@ -671,7 +820,7 @@ func TestClientsJSONFromAnEarlierVersionWithTokenRefusalsLoads(t *testing.T) {
}
}
func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
func TestListsAndDNSBLJSONEntryWithoutAFieldItNeedsIsSetAside(t *testing.T) {
t.Parallel()
const (
@@ -686,76 +835,76 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
)
for _, tc := range []struct {
name, content string
name, file, content string
// want is what the error says after the file's path.
want string
}{
{
"a list without its URL",
"a list without its URL", listsJSON,
`{"version": 1, "lists": [{` + tried + fetched + `, "lines": []}]}`,
`: entry 1 has no "url"`,
},
{
"a list without the time it was last tried",
"a list without the time it was last tried", listsJSON,
`{"version": 1, "lists": [{` + drop + fetched + `, "lines": []}]}`,
`: entry 1 has no "tried"`,
},
{
"a copy of a list without the time it was fetched",
"a copy of a list without the time it was fetched", listsJSON,
`{"version": 1, "lists": [{` + drop + tried + `"lines": []}]}`,
`: entry 1 has no "fetched"`,
noFetched,
},
{
// An empty list has no lines, which is not having none.
"a copy of a list without its lines",
"a copy of a list without its lines", listsJSON,
`{"version": 1, "lists": [{` + drop + tried + fetched + `, "lines": []}, ` +
`{"url": "` + torURL + `", ` + tried + fetched + `}]}`,
`: entry 2 has no "lines"`,
},
{
"a verdict without its zone",
"a verdict without its zone", dnsblJSON,
`{"version": 1, "verdicts": [{` + verdictClient + listed + fetched + `}]}`,
`: verdicts entry 1 has no "zone"`,
`: entry 1 has no "zone"`,
},
{
"a verdict without its client",
"a verdict without its client", dnsblJSON,
`{"version": 1, "verdicts": [{` + verdictZone + listed + fetched + `}]}`,
`: verdicts entry 1 has no "client"`,
noClient,
},
{
// A client the zone does not list has a listed of false, which is
// not having none.
"a verdict without whether the zone lists the client",
"a verdict without whether the zone lists the client", dnsblJSON,
`{"version": 1, "verdicts": [{` + verdictZone + verdictClient + listed +
fetched + `}, {` + verdictZone + verdictClient + fetched + `}]}`,
`: verdicts entry 2 has no "listed"`,
`: entry 2 has no "listed"`,
},
{
"a verdict without the time it was fetched",
"a verdict without the time it was fetched", dnsblJSON,
`{"version": 1, "verdicts": [{` + verdictZone + verdictClient +
`"listed": true}]}`,
`: verdicts entry 1 has no "fetched"`,
noFetched,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantRefused(t, reputationJSON, tc.content, tc.want)
wantSetAside(t, tc.file, tc.content, tc.want)
})
}
}
func TestReputationJSONScoreWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
func TestAbuseIPDBScoresJSONScoreWithoutAFieldItNeedsIsSetAside(t *testing.T) {
t.Parallel()
// scores opens the list of AbuseIPDB scores, and ends closes it; client,
// score and fetched make a score.
const (
scores = `{"version": 1, "abuseipdb": {"scores": [`
scores = `{"version": 1, "scores": [`
client = `"client": "198.51.100.7/32", `
score = `"score": 0, `
fetched = `"fetched": "2026-10-06T00:00:00Z"`
ends = `}]}}`
ends = `}]}`
)
for _, tc := range []struct {
@@ -765,23 +914,23 @@ func TestReputationJSONScoreWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
}{
{
"without its client", scores + `{` + score + fetched + ends,
`: abuseipdb scores entry 1 has no "client"`,
noClient,
},
{
// A score of 0 is not having none.
"without the score",
scores + `{` + client + score + fetched + `}, {` + client + fetched + ends,
`: abuseipdb scores entry 2 has no "score"`,
`: entry 2 has no "score"`,
},
{
"without the time it was fetched", scores + `{` + client + `"score": 100` + ends,
`: abuseipdb scores entry 1 has no "fetched"`,
noFetched,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantRefused(t, reputationJSON, tc.content, tc.want)
wantSetAside(t, abuseIPDBScoresJSON, tc.content, tc.want)
})
}
}
@@ -906,9 +1055,7 @@ func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
func TestUnknownVersionStopsTheStart(t *testing.T) {
t.Parallel()
for _, file := range []string{
bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
} {
for _, file := range []string{bansJSON, clientsJSON, abuseIPDBChecksJSON, alertsJSON} {
for _, content := range []string{`{"version": 2}`, `{}`} {
t.Run(file+" "+content, func(t *testing.T) {
t.Parallel()
@@ -939,6 +1086,17 @@ func TestUnwritableDirectoryStopsTheStart(t *testing.T) {
if err == nil || !strings.HasPrefix(err.Error(), want) {
t.Errorf("state directory %s: error %v, want one starting %s", dir, err, want)
}
const wantCache = "SWW_CACHE_DIR cannot be written: "
params := newParams(t.TempDir())
params.CacheDir = dir
_, err = state.Load(params)
if err == nil || !strings.HasPrefix(err.Error(), wantCache) {
t.Errorf("cache directory %s: error %v, want one starting %s", dir, err,
wantCache)
}
}
}
@@ -1010,9 +1168,8 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) {
time.Sleep(time.Nanosecond)
synctest.Wait()
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON,
reputationJSON)
wantFiles(t, dir, everyFile()...)
removeFiles(t, dir, everyFile()...)
}
})
}
@@ -1231,7 +1388,7 @@ func TestFileThatCannotBeReadIsNotWrittenOver(t *testing.T) {
t.Errorf("bans.json is now %v (%v), want the socket", info, err)
}
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
wantFiles(t, dir, everyFile()...)
wantWriteFailed(t, params, bansJSON)
}
@@ -1304,32 +1461,6 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
wantEqual(t, lookupsJSON, params.GeoJS.Snapshot(),
[]lookup.Answer{{Client: client, Country: "FR", Answered: midnight()}})
edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
`"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+
`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+
`"abuseipdb": {"day": "2026-10-06", "spent": 9, "scores": [`+
`{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`)
wantTakenIn(t, lines, dir, reputationJSON)
listedBy := params.Lists.ListedBy(client.Addr())
if !slices.Equal(listedBy, []string{blocklistURL}) {
t.Errorf("%s taken in lists %s on %v, want on the blocklist", reputationJSON,
client.Addr(), listedBy)
}
wantEqual(t, reputationJSON, params.DNSBL.Snapshot(), []reputation.Verdict{{
Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(),
}})
checks := reputation.Checks{
Day: civil.DateOf(midnight()), Spent: 9,
Scores: []reputation.Score{{Client: client, Score: 80, Fetched: midnight()}},
}
if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) {
t.Errorf("%s taken in as\n%+v\nwant\n%+v", reputationJSON, got, checks)
}
// A netblock with bits past its length is read as the netblock it is
// in.
edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
@@ -1352,6 +1483,59 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
}
}
func TestEditOfEachReputationFileTakenIn(t *testing.T) {
t.Parallel()
dir := t.TempDir()
params := newParams(dir)
lines := logInto(&params)
fill(params)
files := load(t, params)
err := files.WriteAll()
if err != nil {
t.Fatalf("write: %v", err)
}
watch(t, files, lines)
// Each edit holds one entry, for a client the parts did not hold, and
// takes the place of everything the part held.
client := netip.MustParsePrefix("198.51.100.7/32")
edit(t, dir, listsJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
`"lines": ["198.51.100.7"]}]}`)
wantTakenIn(t, lines, dir, listsJSON)
listedBy := params.Lists.ListedBy(client.Addr())
if !slices.Equal(listedBy, []string{blocklistURL}) {
t.Errorf("%s taken in lists %s on %v, want on the blocklist", listsJSON,
client.Addr(), listedBy)
}
edit(t, dir, dnsblJSON, `{"version": 1, "verdicts": [{"zone": "`+dnsblZone+`", `+
`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}]}`)
wantTakenIn(t, lines, dir, dnsblJSON)
wantEqual(t, dnsblJSON, params.DNSBL.Snapshot(), []reputation.Verdict{{
Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(),
}})
edit(t, dir, abuseIPDBScoresJSON, `{"version": 1, "scores": [`+
`{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}`)
wantTakenIn(t, lines, dir, abuseIPDBScoresJSON)
wantEqual(t, abuseIPDBScoresJSON, params.AbuseIPDB.Snapshot(),
[]reputation.Score{{Client: client, Score: 80, Fetched: midnight()}})
edit(t, dir, abuseIPDBChecksJSON, `{"version": 1, "day": "2026-10-06", "spent": 9}`)
wantTakenIn(t, lines, dir, abuseIPDBChecksJSON)
checks := reputation.ChecksSpent{Day: civil.DateOf(midnight()), Spent: 9}
if got := params.AbuseIPDB.ChecksSpent(); got != checks {
t.Errorf("%s taken in as %+v, want %+v", abuseIPDBChecksJSON, got, checks)
}
}
func TestTimesEditedWithAnOffsetWrittenBackInUTC(t *testing.T) {
t.Parallel()
@@ -1374,8 +1558,10 @@ func TestTimesEditedWithAnOffsetWrittenBackInUTC(t *testing.T) {
ahead := time.FixedZone("UTC+2", 2*60*60)
written := map[string]string{}
// abuseipdb-checks.json holds a day, but no time.
for _, name := range []string{
bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
bansJSON, clientsJSON, alertsJSON, lookupsJSON, listsJSON, dnsblJSON,
abuseIPDBScoresJSON,
} {
written[name] = readFile(t, filepath.Join(dir, name))
edited := stamp.ReplaceAllStringFunc(written[name], func(quoted string) string {
@@ -1613,7 +1799,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
edit(t, dir, bansJSON, broken)
edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`)
wantTakenIn(t, lines, dir, clientsJSON)
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
wantFiles(t, dir, everyFile()...)
// The next write sets it aside, logged with where the error is, and
// writes bans.json again from what sww still holds.
@@ -1622,7 +1808,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
t.Fatalf("write: %v", err)
}
line := lines.waitFor(t, "set aside an edit of a state file that does not parse")
line := lines.waitFor(t, setAside)
message, _ := line["error"].(string)
if line["file"] != path+".bad" ||
@@ -1637,8 +1823,8 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad")
}
wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON,
reputationJSON)
wantFiles(t, dir, abuseIPDBChecksJSON, abuseIPDBScoresJSON, alertsJSON, bansJSON,
bansJSON+".bad", clientsJSON, dnsblJSON, listsJSON, lookupsJSON)
if got := readFile(t, path+".bad"); got != broken {
t.Errorf("bans.json.bad holds\n%s\nwant the edit", got)
@@ -1766,9 +1952,9 @@ func midnight() time.Time {
return time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
}
// newParams returns Params for the state files in dir, with parts that
// hold nothing yet. GeoJS is never asked, the lists, two blocklists, are
// never fetched, and the alerts, at most two an hour, are
// newParams returns Params for the state and cache files, both in dir,
// with parts that hold nothing yet. GeoJS is never asked, the lists, two
// blocklists, are never fetched, and the alerts, at most two an hour, are
// never sent. The anomaly counters count the scopes fill counts, with
// thresholds fill does not reach.
func newParams(dir string) state.Params {
@@ -1785,6 +1971,7 @@ func newParams(dir string) state.Params {
return state.Params{
Dir: dir,
CacheDir: dir,
WriteDelay: time.Hour,
CounterInterval: time.Hour,
Ledger: bans.New(bans.Params{
@@ -1837,9 +2024,9 @@ func office() netip.Prefix {
// with counts and histories,
// GeoJS answers, the blocklists' last tries and the copy of one, two
// verdicts of a DNSBL zone, and the AbuseIPDB checks spent today with two
// scores, as filledReputationJSON holds them, and alerts
// and anomaly counters, as filledAlertsJSON holds them, into the parts of
// params.
// scores, as filledListsJSON, filledDNSBLJSON, filledAbuseIPDBScoresJSON
// and filledAbuseIPDBChecksJSON hold them, and alerts and anomaly
// counters, as filledAlertsJSON holds them, into the parts of params.
func fill(params state.Params) {
now := midnight()
client := netip.MustParsePrefix("203.0.113.9/32")
@@ -1895,11 +2082,12 @@ func fill(params state.Params) {
},
{Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)},
})
params.AbuseIPDB.Load(reputation.Checks{
Day: civil.DateOf(now), Spent: 3, Scores: []reputation.Score{
{Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)},
{Client: client, Score: 100, Fetched: now.Add(-time.Hour)},
},
params.AbuseIPDB.Load([]reputation.Score{
{Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)},
{Client: client, Score: 100, Fetched: now.Add(-time.Hour)},
})
params.AbuseIPDB.LoadChecksSpent(reputation.ChecksSpent{
Day: civil.DateOf(now), Spent: 3,
})
// An alert waiting, a repeat of it the cooldown holds back, another
@@ -2160,6 +2348,40 @@ func wantRefused(t *testing.T, file, content, want string) {
}
}
// wantSetAside writes content to the cache file named file in a new
// directory, and checks that Load starts all the same, with the cache
// empty, and sets the file aside, logging an error that is the file's
// path and then starts with want.
func wantSetAside(t *testing.T, file, content, want string) {
t.Helper()
dir := t.TempDir()
path := filepath.Join(dir, file)
err := os.WriteFile(path, []byte(content), 0o600)
if err != nil {
t.Fatalf("write %s: %v", file, err)
}
params := newParams(dir)
lines := logInto(&params)
load(t, params)
line := lines.waitFor(t, setAside)
if message, _ := line["error"].(string); line["file"] != path+".bad" ||
!strings.HasPrefix(message, path+want) {
t.Errorf("set aside with %v, want %s.bad and an error starting %s%s", line, path,
path, want)
}
if len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
len(params.DNSBL.Snapshot()) != 0 || len(params.AbuseIPDB.Snapshot()) != 0 {
t.Error("a cache holds what a file set aside held")
}
wantFiles(t, dir, file+".bad")
}
// removeFiles removes the named files from dir.
func removeFiles(t *testing.T, dir string, names ...string) {
t.Helper()
+1
View File
@@ -244,6 +244,7 @@ func loadStateFiles(
) (*state.Files, error) {
return state.Load(state.Params{
Dir: cfg.StateDir,
CacheDir: cfg.CacheDir,
WriteDelay: cfg.StateWriteDelay,
CounterInterval: cfg.StateCounterInterval,
Ledger: server.Ledger,
+52 -17
View File
@@ -38,6 +38,7 @@ const (
stateDir = "SWW_STATE_DIR"
stateWriteDelay = "SWW_STATE_WRITE_DELAY"
stateCounterInterval = "SWW_STATE_COUNTER_INTERVAL"
cacheDir = "SWW_CACHE_DIR"
rateLimitPerDay = "SWW_RATE_LIMIT_PER_DAY"
rateLimitExemptNets = "SWW_RATE_LIMIT_EXEMPT_NETS"
rulesDir = "SWW_RULES_DIR"
@@ -160,6 +161,12 @@ func run(ctx context.Context, env map[string]string, out *output) int {
return "off", true
}
// The cache files go beside the state files unless the test
// says otherwise.
if !ok && name == cacheDir {
return env[stateDir], env[stateDir] != ""
}
return value, ok
},
Stdout: out,
@@ -699,7 +706,7 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
}
}
func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T) {
func TestBlocklistTriesAndCopiesKeptInListsJSONAcrossRestarts(t *testing.T) {
t.Parallel()
const (
@@ -732,10 +739,11 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
t.Cleanup(server.Close)
torURL, dropURL := server.URL+torPath, server.URL+dropPath
dir := t.TempDir()
cache := t.TempDir()
env := map[string]string{
listenAddr: localhost + ":0",
stateDir: dir,
stateDir: t.TempDir(),
cacheDir: cache,
rulesDir: t.TempDir(),
metricsToken: token,
instanceName: instance,
@@ -755,7 +763,7 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
})
// Restarted with drop.txt named after it and the server answering, tor.txt
// waits SWW_BLOCKLIST_REFRESH after its failed try, kept in reputation.json,
// waits SWW_BLOCKLIST_REFRESH after its failed try, kept in lists.json,
// while drop.txt, never tried, is fetched at once. Lists are fetched in the
// order named, so once drop.txt refuses the client, tor.txt has had its turn.
failing.Store(false)
@@ -773,8 +781,8 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
}
// After another restart, with the server failing, the copy of drop.txt
// kept in reputation.json, its copyright line included, refuses the
// client from the first request.
// kept in lists.json, its copyright line included, refuses the client
// from the first request.
failing.Store(true)
out = runUntilStopped(t, env, func(url string) {
@@ -782,16 +790,16 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
})
wantDeniedByList(t, out.line(t, "type", "request"), dropURL)
path := filepath.Join(dir, "reputation.json")
path := filepath.Join(cache, "lists.json")
kept, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory
if err != nil || !strings.Contains(string(kept), `"`+copyright+`"`) {
t.Errorf("reputation.json holds\n%s\nwant the copy with %q (%v)", kept, copyright,
t.Errorf("lists.json holds\n%s\nwant the copy with %q (%v)", kept, copyright,
err)
}
}
func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
func TestCrowdSecDecisionListKeptInListsJSONAcrossARestart(t *testing.T) {
t.Parallel()
const bouncerKey = "crowdsec-key-0123456789abcdef"
@@ -814,10 +822,11 @@ func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
}))
t.Cleanup(engine.Close)
dir := t.TempDir()
dir, cache := t.TempDir(), t.TempDir()
env := map[string]string{
listenAddr: localhost + ":0",
stateDir: dir,
cacheDir: cache,
rulesDir: t.TempDir(),
"SWW_CROWDSEC_LAPI_URL": engine.URL,
"SWW_CROWDSEC_LAPI_KEY": bouncerKey,
@@ -836,7 +845,7 @@ func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
t.Errorf("bans.json holds %v, want the ban for crowdsec on %s", ban, placed)
}
// Restarted with the engine down, the copy kept in reputation.json bans
// Restarted with the engine down, the copy kept in lists.json bans
// another client in the netblock from its first request.
down.Store(true)
@@ -844,22 +853,25 @@ func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
wantStatus(t, url, "203.0.113.10", http.StatusForbidden)
})
// The key is in neither run's output, nor in a state file.
files := []string{"bans.json", "reputation.json", "clients.json"}
// The key is in neither run's output, nor in a state or cache file.
files := []string{
filepath.Join(dir, "bans.json"), filepath.Join(dir, "clients.json"),
filepath.Join(cache, "lists.json"),
}
shown := make([]string, 0, len(files)+2)
shown = append(shown, first.text(), second.text())
for _, name := range files {
data, err := os.ReadFile(filepath.Join(dir, name)) //nolint:gosec // the test's
for _, path := range files {
data, err := os.ReadFile(path) //nolint:gosec // the test's
if err != nil {
t.Fatalf("read %s: %v", name, err)
t.Fatalf("read %s: %v", path, err)
}
shown = append(shown, string(data))
}
if all := strings.Join(shown, "\n"); strings.Contains(all, bouncerKey) {
t.Errorf("the key is shown in the output or the state files:\n%s", all)
t.Errorf("the key is shown in the output or the state and cache files:\n%s", all)
}
}
@@ -1207,6 +1219,29 @@ func TestUnwritableStateDirStopsTheStart(t *testing.T) {
"SWW_STATE_DIR cannot be written: ")
}
func TestUnwritableCacheDirStopsTheStart(t *testing.T) {
t.Parallel()
ctx, stop := context.WithTimeout(t.Context(), waitLimit)
defer stop()
out := &output{}
status := run(ctx, map[string]string{
listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
cacheDir: filepath.Join(t.TempDir(), "missing"),
}, out)
if status != 1 {
t.Fatalf("exit status %d, want 1", status)
}
line := out.line(t, "msg", "cannot use the state files")
if message, _ := line["error"].(string); !strings.HasPrefix(message,
"SWW_CACHE_DIR cannot be written: ") {
t.Errorf("start refused with %q, want SWW_CACHE_DIR named", message)
}
}
// wantStartRefused runs sww with its state files in dir, and
// checks that it stops at start, with an error that starts with want. If
// it starts instead, it is stopped after waitLimit.
+1
View File
@@ -67,6 +67,7 @@ func TestMainRunsWAF(t *testing.T) {
commandForMain+"="+wafCommand,
listenAddr+"="+localhost+":0",
stateDir+"="+dir,
cacheDir+"="+dir,
rulesDir+"="+t.TempDir(),
lookupSource+"=off")
+2 -2
View File
@@ -759,7 +759,7 @@ func TestOnlyAClientThatHasCommittedAnOffenceIsCheckedWithAbuseIPDB(t *testing.T
clk.advance(time.Hour)
wantReputation(t, s.get(fromDE, http.StatusOK, forward))
wantAbuseIPDBChecks(t, server, 1)
waitUntil(func() bool { return len(server.AbuseIPDB.Snapshot().Scores) == 1 })
waitUntil(func() bool { return len(server.AbuseIPDB.Snapshot()) == 1 })
wantReputation(t, s.get(fromDE, http.StatusOK, forward), abuseipdb)
s.get(fromKP, http.StatusOK, forward)
@@ -1102,7 +1102,7 @@ func loadScores(server *waf.Server, scores map[string]int64) {
})
}
server.AbuseIPDB.Load(reputation.Checks{Scores: kept})
server.AbuseIPDB.Load(kept)
}
// wantAbuseIPDBChecks checks how many clients server has checked with