Blocklists and an AS percentage file fetched by URL (closes #29)
check / check (push) Waiting to run
check / check (push) Waiting to run
SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands for its IPv4 address or netblock. reputation.json keeps each list's last try, failed or not, even one cut off by a stop, which a restart waits on as a running instance does, and its last good copy, whole, used while a fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed client; the log line names the lists, each raises reputation_hit, and a failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning. Judgement call: a failed fetch is retried after the refresh, not sooner. Not done: ban notes do not name the lists yet. Model: opus-5-5
This commit is contained in:
+124
-56
@@ -1,9 +1,10 @@
|
||||
// Package state keeps smallwebwaf's state in JSON files in
|
||||
// SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
|
||||
// bans.json holds the bans, clients.json each client's counters and
|
||||
// history, lookups.json GeoJS's answers, and alerts.json the cooldowns,
|
||||
// the hour under way, the alerts waiting for each destination and the
|
||||
// anomaly counters. Load
|
||||
// history, lookups.json GeoJS's answers, reputation.json the last try and
|
||||
// last good copy of each list fetched from a URL, and alerts.json the
|
||||
// cooldowns, the hour under way, the alerts waiting for each destination
|
||||
// and the anomaly counters. Load
|
||||
// reads them at start, Watch takes in an admin's edit of one while
|
||||
// smallwebwaf runs, and Run and WriteAll write them. The disk is read and
|
||||
// written outside the parts' locks, which are held only to take a
|
||||
@@ -35,6 +36,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// version is the version of the files' format, the only one read.
|
||||
@@ -46,10 +48,11 @@ const fileMode = 0o600
|
||||
|
||||
// The state files' names.
|
||||
const (
|
||||
bansJSON = "bans.json"
|
||||
clientsJSON = "clients.json"
|
||||
lookupsJSON = "lookups.json"
|
||||
alertsJSON = "alerts.json"
|
||||
bansJSON = "bans.json"
|
||||
clientsJSON = "clients.json"
|
||||
lookupsJSON = "lookups.json"
|
||||
reputationJSON = "reputation.json"
|
||||
alertsJSON = "alerts.json"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -73,12 +76,13 @@ type Params struct {
|
||||
// is (SWWAF_STATE_COUNTER_INTERVAL).
|
||||
WriteDelay time.Duration
|
||||
CounterInterval time.Duration
|
||||
// Ledger, Limiter, GeoJS, Alerts and Anomalies hold the state. Alerts
|
||||
// also receive a file_error alert for an edit set aside, and for a
|
||||
// write that fails while smallwebwaf runs.
|
||||
// Ledger, Limiter, GeoJS, Lists, Alerts and Anomalies hold the state.
|
||||
// Alerts also receive a file_error alert for an edit set aside, and for
|
||||
// a write that fails while smallwebwaf runs.
|
||||
Ledger *bans.Ledger
|
||||
Limiter *ratelimit.Limiter
|
||||
GeoJS *lookup.GeoJS
|
||||
Lists *reputation.Lists
|
||||
Alerts *alerts.Queue
|
||||
Anomalies *anomaly.Counters
|
||||
// Now tells the time by which the counters' buckets run out, normally
|
||||
@@ -138,6 +142,13 @@ type lookupsFile struct {
|
||||
Lookups []lookup.Answer `json:"lookups"`
|
||||
}
|
||||
|
||||
// reputationFile is reputation.json, indented for an admin to read and
|
||||
// edit, so that each line of a list's copy is on a line of its own.
|
||||
type reputationFile struct {
|
||||
Version int `json:"version"`
|
||||
Lists []reputation.List `json:"lists"`
|
||||
}
|
||||
|
||||
// alertsFile is alerts.json, indented for an admin to read and edit.
|
||||
//
|
||||
//nolint:tagliatelle // the state files use snake_case, as the request log does
|
||||
@@ -159,10 +170,10 @@ type stateFile interface {
|
||||
}
|
||||
|
||||
// Load checks that files can be written in Dir, and reads the state files
|
||||
// in it into the ledger, the limiter and GeoJS. A missing file is empty
|
||||
// state, as on a first start. A file that does not parse, has an unknown
|
||||
// version, or has an entry without a field it needs, is an error that
|
||||
// names the file and, where the JSON decoder tells it, the line and
|
||||
// in it into the parts of Params that hold the state. A missing file is
|
||||
// empty state, as on a first start. A file that does not parse, has an
|
||||
// unknown version, or has an entry without a field it needs, is an error
|
||||
// that names the file and, where the JSON decoder tells it, the line and
|
||||
// column, or else the entry.
|
||||
func Load(params Params) (*Files, error) {
|
||||
err := checkWritable(params.Dir)
|
||||
@@ -175,16 +186,17 @@ func Load(params Params) (*Files, error) {
|
||||
bansRead, bansErr := f.read(bansJSON)
|
||||
clientsRead, clientsErr := f.read(clientsJSON)
|
||||
lookupsRead, lookupsErr := f.read(lookupsJSON)
|
||||
reputationRead, reputationErr := f.read(reputationJSON)
|
||||
alertsRead, alertsErr := f.read(alertsJSON)
|
||||
|
||||
err = errors.Join(bansErr, clientsErr, lookupsErr, alertsErr)
|
||||
err = errors.Join(bansErr, clientsErr, lookupsErr, reputationErr, alertsErr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
params.ProcessLog.Info("read the state files", "directory", params.Dir,
|
||||
"bans", bansRead, "clients", clientsRead, "lookups", lookupsRead,
|
||||
"alerts_waiting", alertsRead)
|
||||
"lists", reputationRead, "alerts_waiting", alertsRead)
|
||||
|
||||
return f, nil
|
||||
}
|
||||
@@ -213,7 +225,9 @@ func (f *Files) Run(ctx context.Context) {
|
||||
|
||||
f.logFailure(bansJSON, f.writeFile(bansJSON))
|
||||
case <-interval.C:
|
||||
for _, name := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} {
|
||||
for _, name := range []string{
|
||||
bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
|
||||
} {
|
||||
f.logFailure(name, f.writeFile(name))
|
||||
}
|
||||
}
|
||||
@@ -224,7 +238,7 @@ func (f *Files) Run(ctx context.Context) {
|
||||
// fails does not keep the others from being written.
|
||||
func (f *Files) WriteAll() error {
|
||||
return errors.Join(f.writeFile(bansJSON), f.writeFile(clientsJSON),
|
||||
f.writeFile(lookupsJSON), f.writeFile(alertsJSON))
|
||||
f.writeFile(lookupsJSON), f.writeFile(reputationJSON), f.writeFile(alertsJSON))
|
||||
}
|
||||
|
||||
// Watch watches Dir until ctx is done, and takes in an admin's edit of a
|
||||
@@ -259,7 +273,7 @@ func (f *Files) Watch(ctx context.Context) {
|
||||
return
|
||||
case event := <-watcher.Events:
|
||||
switch name := filepath.Base(event.Name); name {
|
||||
case bansJSON, clientsJSON, lookupsJSON, alertsJSON:
|
||||
case bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON:
|
||||
f.fileChanged(name)
|
||||
}
|
||||
case err = <-watcher.Errors:
|
||||
@@ -405,33 +419,27 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
|
||||
|
||||
f.params.GeoJS.Load(file.Lookups)
|
||||
entries = len(file.Lookups)
|
||||
case alertsJSON:
|
||||
// waiting was a list, of the alerts waiting for the webhook, before
|
||||
// alerts went to Slack and ntfy too.
|
||||
var written struct {
|
||||
Waiting json.RawMessage `json:"waiting"`
|
||||
}
|
||||
|
||||
if json.Unmarshal(data, &written) == nil &&
|
||||
bytes.HasPrefix(written.Waiting, []byte("[")) {
|
||||
return 0, fmt.Errorf("%s: %w", path, errWaitingList)
|
||||
}
|
||||
|
||||
var file alertsFile
|
||||
case reputationJSON:
|
||||
var file reputationFile
|
||||
|
||||
err := parse(path, data, &file)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
f.params.Alerts.Load(alerts.State{
|
||||
Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting,
|
||||
})
|
||||
f.params.Anomalies.Load(file.AnomalyCounters, f.params.Now())
|
||||
|
||||
for _, waiting := range file.Waiting {
|
||||
entries += len(waiting)
|
||||
err = f.params.Lists.Load(file.Lists)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
|
||||
entries = len(file.Lists)
|
||||
case alertsJSON:
|
||||
waiting, err := f.takeInAlerts(path, data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
entries = waiting
|
||||
}
|
||||
|
||||
f.sums[name] = sha256.Sum256(data)
|
||||
@@ -439,6 +447,41 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// takeInAlerts parses data, what alerts.json, at path, holds, puts it
|
||||
// into the alerts and the anomaly counters, in place of what they held,
|
||||
// and returns how many alerts wait in it, as takeIn describes.
|
||||
func (f *Files) takeInAlerts(path string, data []byte) (int, error) {
|
||||
// waiting was a list, of the alerts waiting for the webhook, before
|
||||
// alerts went to Slack and ntfy too.
|
||||
var written struct {
|
||||
Waiting json.RawMessage `json:"waiting"`
|
||||
}
|
||||
|
||||
if json.Unmarshal(data, &written) == nil &&
|
||||
bytes.HasPrefix(written.Waiting, []byte("[")) {
|
||||
return 0, fmt.Errorf("%s: %w", path, errWaitingList)
|
||||
}
|
||||
|
||||
var file alertsFile
|
||||
|
||||
err := parse(path, data, &file)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
f.params.Alerts.Load(alerts.State{
|
||||
Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting,
|
||||
})
|
||||
f.params.Anomalies.Load(file.AnomalyCounters, f.params.Now())
|
||||
|
||||
entries := 0
|
||||
for _, waiting := range file.Waiting {
|
||||
entries += len(waiting)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// writeFile writes the state file name from what smallwebwaf holds. An
|
||||
// edit made since smallwebwaf last read or wrote the file is taken in
|
||||
// first, so that it is not overwritten, or set aside if it does not
|
||||
@@ -514,34 +557,38 @@ func (f *Files) setAside(name string, parseErr error) error {
|
||||
func (f *Files) encode(name string) ([]byte, error) {
|
||||
switch name {
|
||||
case bansJSON:
|
||||
file := bansFile{Version: version, Bans: BanEntries(f.params.Ledger.Snapshot())}
|
||||
|
||||
data, err := json.MarshalIndent(file, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return append(data, '\n'), nil
|
||||
return encodeIndented(bansFile{
|
||||
Version: version, Bans: BanEntries(f.params.Ledger.Snapshot()),
|
||||
})
|
||||
case clientsJSON:
|
||||
return encodeOnePerLine("clients", f.params.Limiter.Snapshot())
|
||||
case lookupsJSON:
|
||||
return encodeOnePerLine("lookups", f.params.GeoJS.Snapshot())
|
||||
case reputationJSON:
|
||||
return encodeIndented(reputationFile{
|
||||
Version: version, Lists: f.params.Lists.Snapshot(),
|
||||
})
|
||||
default: // alerts.json
|
||||
held := f.params.Alerts.Snapshot()
|
||||
file := alertsFile{
|
||||
|
||||
return encodeIndented(alertsFile{
|
||||
Version: version, Cooldowns: held.Cooldowns, Hour: held.Hour,
|
||||
Waiting: held.Waiting, AnomalyCounters: f.params.Anomalies.Snapshot(),
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(file, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return append(data, '\n'), nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// encodeIndented encodes file, a state file's struct, indented for an
|
||||
// admin to read and edit.
|
||||
func encodeIndented(file any) ([]byte, error) {
|
||||
data, err := json.MarshalIndent(file, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return append(data, '\n'), nil
|
||||
}
|
||||
|
||||
// BanEntries returns held as bans.json lists them, an empty list for
|
||||
// none.
|
||||
func BanEntries(held []bans.Ban) []BanEntry {
|
||||
@@ -679,6 +726,27 @@ func (f *lookupsFile) check(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// check refuses a list without its URL, which would name no list, or the
|
||||
// time it was last tried, which would have it fetched at once, and a copy
|
||||
// of it without the time it was fetched, or without its lines, which hold
|
||||
// the list.
|
||||
func (f *reputationFile) check([]byte) error {
|
||||
for i, kept := range f.Lists {
|
||||
switch {
|
||||
case kept.URL == "":
|
||||
return missing(i, "url")
|
||||
case kept.Tried.IsZero():
|
||||
return missing(i, "tried")
|
||||
case kept.Fetched.IsZero() && kept.Lines != nil:
|
||||
return missing(i, "fetched")
|
||||
case kept.Lines == nil && !kept.Fetched.IsZero():
|
||||
return missing(i, "lines")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// check refuses a cooldown without its event or when its alert was sent,
|
||||
// which would hold back no repeat, alerts waiting for a destination with
|
||||
// another name than webhook, slack or ntfy, most likely misspelt, an
|
||||
|
||||
+161
-19
@@ -27,15 +27,20 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||
)
|
||||
|
||||
const (
|
||||
// The state files.
|
||||
bansJSON = "bans.json"
|
||||
clientsJSON = "clients.json"
|
||||
lookupsJSON = "lookups.json"
|
||||
alertsJSON = "alerts.json"
|
||||
bansJSON = "bans.json"
|
||||
clientsJSON = "clients.json"
|
||||
lookupsJSON = "lookups.json"
|
||||
reputationJSON = "reputation.json"
|
||||
alertsJSON = "alerts.json"
|
||||
// blocklistURL and torURL are the blocklists the tests' lists name.
|
||||
blocklistURL = "https://lists.example/drop.txt"
|
||||
torURL = "https://lists.example/tor.txt"
|
||||
// The AS number and AS name the tests' clients are looked up in.
|
||||
asn = "AS64496"
|
||||
asName = "Example Net"
|
||||
@@ -204,6 +209,29 @@ const filledAlertsJSON = `{
|
||||
}
|
||||
`
|
||||
|
||||
// filledReputationJSON is reputation.json holding the blocklists' last
|
||||
// tries and the copy of one, with its comment line, as fill puts them in.
|
||||
const filledReputationJSON = `{
|
||||
"version": 1,
|
||||
"lists": [
|
||||
{
|
||||
"url": "https://lists.example/drop.txt",
|
||||
"tried": "2026-10-06T00:00:00Z",
|
||||
"fetched": "2026-10-05T23:00:00Z",
|
||||
"lines": [
|
||||
"; Spamhaus DROP List 2026/10/05 - (c) 2026 The Spamhaus Project SLL",
|
||||
"203.0.113.0/24 ; SBL1",
|
||||
"2001:db8::/32 ; SBL2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"url": "https://lists.example/tor.txt",
|
||||
"tried": "2026-10-06T00:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
`
|
||||
|
||||
func TestFilesWrittenAndReadBack(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -230,6 +258,11 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
|
||||
wantEqual(t, clientsJSON, after.Limiter.Snapshot(), before.Limiter.Snapshot())
|
||||
wantEqual(t, lookupsJSON, after.GeoJS.Snapshot(), before.GeoJS.Snapshot())
|
||||
|
||||
if got, want := after.Lists.Snapshot(), before.Lists.Snapshot(); !reflect.DeepEqual(
|
||||
got, want) {
|
||||
t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, got, want)
|
||||
}
|
||||
|
||||
if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
|
||||
got, want) {
|
||||
t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want)
|
||||
@@ -238,12 +271,32 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
|
||||
wantEqual(t, alertsJSON, after.Anomalies.Snapshot(), before.Anomalies.Snapshot())
|
||||
|
||||
// Each one-per-line file lists its entries by client, and nothing
|
||||
// but the four files is left in the directory.
|
||||
// but the five files is left in the directory.
|
||||
wantEntries(t, filepath.Join(dir, clientsJSON), "clients",
|
||||
"192.0.2.1/32", "203.0.113.9/32", "2001:db8::/64")
|
||||
wantEntries(t, filepath.Join(dir, lookupsJSON), "lookups",
|
||||
"192.0.2.1/32", "203.0.113.9/32")
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
|
||||
}
|
||||
|
||||
func TestReputationJSONKeepsEachCopyWholeOneLineToALine(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
params := newParams(dir)
|
||||
fill(params)
|
||||
|
||||
files := load(t, params)
|
||||
|
||||
err := files.WriteAll()
|
||||
if err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
got := readFile(t, filepath.Join(dir, reputationJSON))
|
||||
if got != filledReputationJSON {
|
||||
t.Errorf("reputation.json\n%s\nwant\n%s", got, filledReputationJSON)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertsJSONIsIndentedWithTheCooldownsTheHourAndTheAlertsWaiting(t *testing.T) {
|
||||
@@ -368,8 +421,9 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
|
||||
|
||||
held := params.Alerts.Snapshot()
|
||||
if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
|
||||
len(params.GeoJS.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
|
||||
len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 {
|
||||
len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
|
||||
len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 ||
|
||||
held.Hour.Sent != 0 {
|
||||
t.Error("state from no files")
|
||||
}
|
||||
}
|
||||
@@ -428,6 +482,14 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
|
||||
`: anomaly_counters entry 2's scope "nett" is not client, net, asn, total ` +
|
||||
`or watch`,
|
||||
},
|
||||
{
|
||||
"a copy of a list with a line that does not read", reputationJSON,
|
||||
`{"version": 1, "lists": [{"url": "` + blocklistURL + `", ` +
|
||||
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", ` +
|
||||
`"lines": ["; DROP", "203.0.113.300"]}]}`,
|
||||
`: the copy of ` + blocklistURL + `: line 2 is not an address or a netblock, ` +
|
||||
`such as 192.0.2.0/24`,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -523,6 +585,51 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
drop = `"url": "` + blocklistURL + `", `
|
||||
tried = `"tried": "2026-10-06T00:00:00Z", `
|
||||
fetched = `"fetched": "2026-10-06T00:00:00Z"`
|
||||
)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name, content string
|
||||
// want is what the error says after the file's path.
|
||||
want string
|
||||
}{
|
||||
{
|
||||
"a list without its URL",
|
||||
`{"version": 1, "lists": [{` + tried + fetched + `, "lines": []}]}`,
|
||||
`: entry 1 has no "url"`,
|
||||
},
|
||||
{
|
||||
"a list without the time it was last tried",
|
||||
`{"version": 1, "lists": [{` + drop + fetched + `, "lines": []}]}`,
|
||||
`: entry 1 has no "tried"`,
|
||||
},
|
||||
{
|
||||
"a copy of a list without the time it was fetched",
|
||||
`{"version": 1, "lists": [{` + drop + tried + `"lines": []}]}`,
|
||||
`: entry 1 has no "fetched"`,
|
||||
},
|
||||
{
|
||||
// An empty list has no lines, which is not having none.
|
||||
"a copy of a list without its lines",
|
||||
`{"version": 1, "lists": [{` + drop + tried + fetched + `, "lines": []}, ` +
|
||||
`{"url": "` + torURL + `", ` + tried + fetched + `}]}`,
|
||||
`: entry 2 has no "lines"`,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
wantRefused(t, reputationJSON, tc.content, tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -626,7 +733,9 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
|
||||
func TestUnknownVersionStopsTheStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, file := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} {
|
||||
for _, file := range []string{
|
||||
bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
|
||||
} {
|
||||
for _, content := range []string{`{"version": 2}`, `{}`} {
|
||||
t.Run(file+" "+content, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -728,8 +837,9 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) {
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
|
||||
removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
|
||||
removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON,
|
||||
reputationJSON)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -948,7 +1058,7 @@ func TestFileThatCannotBeReadIsNotWrittenOver(t *testing.T) {
|
||||
t.Errorf("bans.json is now %v (%v), want the socket", info, err)
|
||||
}
|
||||
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
|
||||
wantWriteFailed(t, params, bansJSON)
|
||||
}
|
||||
|
||||
@@ -1021,6 +1131,17 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
|
||||
wantEqual(t, lookupsJSON, params.GeoJS.Snapshot(),
|
||||
[]lookup.Answer{{Client: client, Country: "FR", Answered: midnight()}})
|
||||
|
||||
edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
|
||||
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
|
||||
`"lines": ["198.51.100.7"]}]}`)
|
||||
wantTakenIn(t, lines, dir, reputationJSON)
|
||||
|
||||
listedBy := params.Lists.ListedBy(client.Addr())
|
||||
if !slices.Equal(listedBy, []string{blocklistURL}) {
|
||||
t.Errorf("%s taken in lists %s on %v, want on the blocklist", reputationJSON,
|
||||
client.Addr(), listedBy)
|
||||
}
|
||||
|
||||
// A netblock with bits past its length is read as the netblock it is
|
||||
// in.
|
||||
edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
|
||||
@@ -1248,7 +1369,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
|
||||
edit(t, dir, bansJSON, broken)
|
||||
edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`)
|
||||
wantTakenIn(t, lines, dir, clientsJSON)
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
|
||||
|
||||
// The next write sets it aside, logged with where the error is, and
|
||||
// writes bans.json again from what smallwebwaf still holds.
|
||||
@@ -1272,7 +1393,8 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
|
||||
t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad")
|
||||
}
|
||||
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON)
|
||||
wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON,
|
||||
reputationJSON)
|
||||
|
||||
if got := readFile(t, path+".bad"); got != broken {
|
||||
t.Errorf("bans.json.bad holds\n%s\nwant the edit", got)
|
||||
@@ -1401,9 +1523,10 @@ func midnight() time.Time {
|
||||
}
|
||||
|
||||
// newParams returns Params for the state files in dir, with parts that
|
||||
// hold nothing yet. GeoJS is never asked, and the alerts, at most two an
|
||||
// hour, are never sent. The anomaly counters count the scopes fill
|
||||
// counts, with thresholds fill does not reach.
|
||||
// hold nothing yet. GeoJS is never asked, the lists, two blocklists, are
|
||||
// never fetched, and the alerts, at most two an hour, are
|
||||
// never sent. The anomaly counters count the scopes fill counts, with
|
||||
// thresholds fill does not reach.
|
||||
func newParams(dir string) state.Params {
|
||||
discard := slog.New(slog.DiscardHandler)
|
||||
m := metrics.New(1, "app")
|
||||
@@ -1431,6 +1554,10 @@ func newParams(dir string) state.Params {
|
||||
GeoJS: lookup.New(lookup.Params{
|
||||
Now: midnight, ProcessLog: discard, Metrics: m,
|
||||
}),
|
||||
Lists: reputation.New(reputation.Params{
|
||||
BlocklistURLs: []string{blocklistURL, torURL}, Refresh: 24 * time.Hour,
|
||||
Now: midnight, ProcessLog: discard, Alerts: queue,
|
||||
}),
|
||||
Alerts: queue,
|
||||
Anomalies: anomaly.New(anomaly.Params{
|
||||
Net: anomaly.Thresholds{RequestsPerMinute: 1000},
|
||||
@@ -1455,8 +1582,9 @@ func office() netip.Prefix {
|
||||
|
||||
// fill puts a permanent ban an admin made, a ban for a broken limit and
|
||||
// one for a clear sign of attack, clients with counts and histories,
|
||||
// GeoJS answers, and alerts and anomaly counters, as filledAlertsJSON
|
||||
// holds them, into the parts of params.
|
||||
// GeoJS answers, the blocklists' last tries and the copy of one, as
|
||||
// filledReputationJSON holds them, and alerts and anomaly counters, as
|
||||
// filledAlertsJSON holds them, into the parts of params.
|
||||
func fill(params state.Params) {
|
||||
now := midnight()
|
||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||
@@ -1489,6 +1617,20 @@ func fill(params state.Params) {
|
||||
},
|
||||
})
|
||||
|
||||
// The copy of drop.txt was fetched an hour ago, and the fetches of it
|
||||
// and of tor.txt tried since failed.
|
||||
err := params.Lists.Load([]reputation.List{{
|
||||
URL: blocklistURL, Tried: now, Fetched: now.Add(-time.Hour),
|
||||
Lines: []string{
|
||||
"; Spamhaus DROP List 2026/10/05 - (c) 2026 The Spamhaus Project SLL",
|
||||
"203.0.113.0/24 ; SBL1",
|
||||
"2001:db8::/32 ; SBL2",
|
||||
},
|
||||
}, {URL: torURL, Tried: now}})
|
||||
if err != nil {
|
||||
panic(err) // the copy reads
|
||||
}
|
||||
|
||||
// An alert waiting, a repeat of it the cooldown holds back, another
|
||||
// alert waiting, and one past the two an hour, for the hour's summary.
|
||||
ban := alerts.Alert{
|
||||
|
||||
Reference in New Issue
Block a user