Blocklists and an AS percentage file fetched by URL (closes #29)

SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every
SWWAF_BLOCKLIST_REFRESH (24h, never under 1h). The last good copy is kept
whole in reputation.json and used while a fetch fails and across restarts.
SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed client; the log
line names the lists, each raises reputation_hit, and a failed fetch raises
source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched the same way and
counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning.

Judgement call: a restart fetches only lists whose copy is due.
Judgement call: a failed fetch is retried after the refresh, not sooner.
Not done: ban notes do not name the lists yet.

Model: opus-5-5

internal/state/state.go

@@ -1,9 +1,10 @@
 // Package state keeps smallwebwaf's state in JSON files in
 // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
 // bans.json holds the bans, clients.json each client's counters and
-// history, lookups.json GeoJS's answers, and alerts.json the cooldowns,
-// the hour under way, the alerts waiting for each destination and the
-// anomaly counters. Load
+// history, lookups.json GeoJS's answers, reputation.json the last good
+// copy of each list fetched from a URL, and alerts.json the cooldowns, the
+// hour under way, the alerts waiting for each destination and the anomaly
+// counters. Load
 // reads them at start, Watch takes in an admin's edit of one while
 // smallwebwaf runs, and Run and WriteAll write them. The disk is read and
 // written outside the parts' locks, which are held only to take a
@@ -35,6 +36,7 @@ import (
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
+	"sneak.berlin/go/smallwebwaf/internal/reputation"
 )
 
 // version is the version of the files' format, the only one read.
@@ -46,10 +48,11 @@ const fileMode = 0o600
 
 // The state files' names.
 const (
-	bansJSON    = "bans.json"
-	clientsJSON = "clients.json"
-	lookupsJSON = "lookups.json"
-	alertsJSON  = "alerts.json"
+	bansJSON       = "bans.json"
+	clientsJSON    = "clients.json"
+	lookupsJSON    = "lookups.json"
+	reputationJSON = "reputation.json"
+	alertsJSON     = "alerts.json"
 )
 
 var (
@@ -73,12 +76,13 @@ type Params struct {
 	// is (SWWAF_STATE_COUNTER_INTERVAL).
 	WriteDelay      time.Duration
 	CounterInterval time.Duration
-	// Ledger, Limiter, GeoJS, Alerts and Anomalies hold the state. Alerts
-	// also receive a file_error alert for an edit set aside, and for a
-	// write that fails while smallwebwaf runs.
+	// Ledger, Limiter, GeoJS, Lists, Alerts and Anomalies hold the state.
+	// Alerts also receive a file_error alert for an edit set aside, and for
+	// a write that fails while smallwebwaf runs.
 	Ledger    *bans.Ledger
 	Limiter   *ratelimit.Limiter
 	GeoJS     *lookup.GeoJS
+	Lists     *reputation.Lists
 	Alerts    *alerts.Queue
 	Anomalies *anomaly.Counters
 	// Now tells the time by which the counters' buckets run out, normally
@@ -138,6 +142,13 @@ type lookupsFile struct {
 	Lookups []lookup.Answer `json:"lookups"`
 }
 
+// reputationFile is reputation.json, indented for an admin to read and
+// edit, so that each line of a list's copy is on a line of its own.
+type reputationFile struct {
+	Version int               `json:"version"`
+	Lists   []reputation.List `json:"lists"`
+}
+
 // alertsFile is alerts.json, indented for an admin to read and edit.
 //
 //nolint:tagliatelle // the state files use snake_case, as the request log does
@@ -159,10 +170,10 @@ type stateFile interface {
 }
 
 // Load checks that files can be written in Dir, and reads the state files
-// in it into the ledger, the limiter and GeoJS. A missing file is empty
-// state, as on a first start. A file that does not parse, has an unknown
-// version, or has an entry without a field it needs, is an error that
-// names the file and, where the JSON decoder tells it, the line and
+// in it into the parts of Params that hold the state. A missing file is
+// empty state, as on a first start. A file that does not parse, has an
+// unknown version, or has an entry without a field it needs, is an error
+// that names the file and, where the JSON decoder tells it, the line and
 // column, or else the entry.
 func Load(params Params) (*Files, error) {
 	err := checkWritable(params.Dir)
@@ -175,16 +186,17 @@ func Load(params Params) (*Files, error) {
 	bansRead, bansErr := f.read(bansJSON)
 	clientsRead, clientsErr := f.read(clientsJSON)
 	lookupsRead, lookupsErr := f.read(lookupsJSON)
+	reputationRead, reputationErr := f.read(reputationJSON)
 	alertsRead, alertsErr := f.read(alertsJSON)
 
-	err = errors.Join(bansErr, clientsErr, lookupsErr, alertsErr)
+	err = errors.Join(bansErr, clientsErr, lookupsErr, reputationErr, alertsErr)
 	if err != nil {
 		return nil, err
 	}
 
 	params.ProcessLog.Info("read the state files", "directory", params.Dir,
 		"bans", bansRead, "clients", clientsRead, "lookups", lookupsRead,
-		"alerts_waiting", alertsRead)
+		"lists", reputationRead, "alerts_waiting", alertsRead)
 
 	return f, nil
 }
@@ -213,7 +225,9 @@ func (f *Files) Run(ctx context.Context) {
 
 			f.logFailure(bansJSON, f.writeFile(bansJSON))
 		case <-interval.C:
-			for _, name := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} {
+			for _, name := range []string{
+				bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
+			} {
 				f.logFailure(name, f.writeFile(name))
 			}
 		}
@@ -224,7 +238,7 @@ func (f *Files) Run(ctx context.Context) {
 // fails does not keep the others from being written.
 func (f *Files) WriteAll() error {
 	return errors.Join(f.writeFile(bansJSON), f.writeFile(clientsJSON),
-		f.writeFile(lookupsJSON), f.writeFile(alertsJSON))
+		f.writeFile(lookupsJSON), f.writeFile(reputationJSON), f.writeFile(alertsJSON))
 }
 
 // Watch watches Dir until ctx is done, and takes in an admin's edit of a
@@ -259,7 +273,7 @@ func (f *Files) Watch(ctx context.Context) {
 			return
 		case event := <-watcher.Events:
 			switch name := filepath.Base(event.Name); name {
-			case bansJSON, clientsJSON, lookupsJSON, alertsJSON:
+			case bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON:
 				f.fileChanged(name)
 			}
 		case err = <-watcher.Errors:
@@ -405,33 +419,27 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
 
 		f.params.GeoJS.Load(file.Lookups)
 		entries = len(file.Lookups)
-	case alertsJSON:
-		// waiting was a list, of the alerts waiting for the webhook, before
-		// alerts went to Slack and ntfy too.
-		var written struct {
-			Waiting json.RawMessage `json:"waiting"`
-		}
-
-		if json.Unmarshal(data, &written) == nil &&
-			bytes.HasPrefix(written.Waiting, []byte("[")) {
-			return 0, fmt.Errorf("%s: %w", path, errWaitingList)
-		}
-
-		var file alertsFile
+	case reputationJSON:
+		var file reputationFile
 
 		err := parse(path, data, &file)
 		if err != nil {
 			return 0, err
 		}
 
-		f.params.Alerts.Load(alerts.State{
-			Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting,
-		})
-		f.params.Anomalies.Load(file.AnomalyCounters, f.params.Now())
-
-		for _, waiting := range file.Waiting {
-			entries += len(waiting)
+		err = f.params.Lists.Load(file.Lists)
+		if err != nil {
+			return 0, fmt.Errorf("%s: %w", path, err)
 		}
+
+		entries = len(file.Lists)
+	case alertsJSON:
+		waiting, err := f.takeInAlerts(path, data)
+		if err != nil {
+			return 0, err
+		}
+
+		entries = waiting
 	}
 
 	f.sums[name] = sha256.Sum256(data)
@@ -439,6 +447,41 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
 	return entries, nil
 }
 
+// takeInAlerts parses data, what alerts.json, at path, holds, puts it
+// into the alerts and the anomaly counters, in place of what they held,
+// and returns how many alerts wait in it, as takeIn describes.
+func (f *Files) takeInAlerts(path string, data []byte) (int, error) {
+	// waiting was a list, of the alerts waiting for the webhook, before
+	// alerts went to Slack and ntfy too.
+	var written struct {
+		Waiting json.RawMessage `json:"waiting"`
+	}
+
+	if json.Unmarshal(data, &written) == nil &&
+		bytes.HasPrefix(written.Waiting, []byte("[")) {
+		return 0, fmt.Errorf("%s: %w", path, errWaitingList)
+	}
+
+	var file alertsFile
+
+	err := parse(path, data, &file)
+	if err != nil {
+		return 0, err
+	}
+
+	f.params.Alerts.Load(alerts.State{
+		Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting,
+	})
+	f.params.Anomalies.Load(file.AnomalyCounters, f.params.Now())
+
+	entries := 0
+	for _, waiting := range file.Waiting {
+		entries += len(waiting)
+	}
+
+	return entries, nil
+}
+
 // writeFile writes the state file name from what smallwebwaf holds. An
 // edit made since smallwebwaf last read or wrote the file is taken in
 // first, so that it is not overwritten, or set aside if it does not
@@ -514,34 +557,38 @@ func (f *Files) setAside(name string, parseErr error) error {
 func (f *Files) encode(name string) ([]byte, error) {
 	switch name {
 	case bansJSON:
-		file := bansFile{Version: version, Bans: BanEntries(f.params.Ledger.Snapshot())}
-
-		data, err := json.MarshalIndent(file, "", "    ")
-		if err != nil {
-			return nil, err
-		}
-
-		return append(data, '\n'), nil
+		return encodeIndented(bansFile{
+			Version: version, Bans: BanEntries(f.params.Ledger.Snapshot()),
+		})
 	case clientsJSON:
 		return encodeOnePerLine("clients", f.params.Limiter.Snapshot())
 	case lookupsJSON:
 		return encodeOnePerLine("lookups", f.params.GeoJS.Snapshot())
+	case reputationJSON:
+		return encodeIndented(reputationFile{
+			Version: version, Lists: f.params.Lists.Snapshot(),
+		})
 	default: // alerts.json
 		held := f.params.Alerts.Snapshot()
-		file := alertsFile{
+
+		return encodeIndented(alertsFile{
 			Version: version, Cooldowns: held.Cooldowns, Hour: held.Hour,
 			Waiting: held.Waiting, AnomalyCounters: f.params.Anomalies.Snapshot(),
-		}
-
-		data, err := json.MarshalIndent(file, "", "    ")
-		if err != nil {
-			return nil, err
-		}
-
-		return append(data, '\n'), nil
+		})
 	}
 }
 
+// encodeIndented encodes file, a state file's struct, indented for an
+// admin to read and edit.
+func encodeIndented(file any) ([]byte, error) {
+	data, err := json.MarshalIndent(file, "", "    ")
+	if err != nil {
+		return nil, err
+	}
+
+	return append(data, '\n'), nil
+}
+
 // BanEntries returns held as bans.json lists them, an empty list for
 // none.
 func BanEntries(held []bans.Ban) []BanEntry {
@@ -679,6 +726,24 @@ func (f *lookupsFile) check(data []byte) error {
 	return nil
 }
 
+// check refuses a list's copy without its URL, which would name no list,
+// the time it was fetched, which would have the list fetched at once, or
+// its lines, which hold the list.
+func (f *reputationFile) check([]byte) error {
+	for i, kept := range f.Lists {
+		switch {
+		case kept.URL == "":
+			return missing(i, "url")
+		case kept.Fetched.IsZero():
+			return missing(i, "fetched")
+		case kept.Lines == nil:
+			return missing(i, "lines")
+		}
+	}
+
+	return nil
+}
+
 // check refuses a cooldown without its event or when its alert was sent,
 // which would hold back no repeat, alerts waiting for a destination with
 // another name than webhook, slack or ntfy, most likely misspelt, an