Caches of outside APIs live in /var/cache/sww (closes #314)
check / check (push) Waiting to run
check / check (push) Waiting to run
The answers of GeoJS, the lists fetched by URL, the DNS blocklists, AbuseIPDB and routewatch each go in a cache file in SWW_CACHE_DIR (default /var/cache/sww): lookups.json, lists.json, dnsbl.json, abuseipdb-scores.json and routewatch.json. reputation.json is gone. The AbuseIPDB checks spent today are a count, not a cache, and stay in SWW_STATE_DIR as abuseipdb-checks-spent.json. A cache file never stops the start: one that does not parse is set aside as an edit is, one that cannot be read is written over, and its cache starts empty. An unwritable SWW_CACHE_DIR stops the start, as an unwritable SWW_STATE_DIR does. Each directory is watched on its own. The run script makes the cache directory and gives it to sww. Model: opus-5-5
This commit is contained in:
1 parent
6e370b346b
commit
7b523720e8
30 files changed
+1677
-585
No files matched your search
+463
-157
@@ -1,13 +1,23 @@
|
||||
// Package state keeps sww's state in JSON files in
|
||||
// SWW_STATE_DIR, as the "Persistent state" section of SPEC.md requires
|
||||
// and docs/state-files.md lists them:
|
||||
// bans.json holds the bans, clients.json each client's counters and
|
||||
// history, lookups.json GeoJS's answers, reputation.json the last try and
|
||||
// last good copy of each list fetched from a URL, the DNSBL zones'
|
||||
// verdicts, and AbuseIPDB's scores and checks spent, and alerts.json the
|
||||
// cooldowns, the hour under way, the alerts waiting for each destination
|
||||
// and the anomaly counters. Load
|
||||
// reads them at start, Watch takes in an admin's edit of one while
|
||||
// Package state keeps sww's state in JSON files, as the "Persistent
|
||||
// state" section of SPEC.md requires and docs/state-files.md lists them.
|
||||
//
|
||||
// The state files are in SWW_STATE_DIR:
|
||||
// - bans.json holds the bans;
|
||||
// - clients.json holds each client's counters and history;
|
||||
// - abuseipdb-checks-spent.json holds the AbuseIPDB checks spent today;
|
||||
// - alerts.json holds the cooldowns, the hour under way, the alerts
|
||||
// waiting for each destination and the anomaly counters.
|
||||
//
|
||||
// The cache files are in SWW_CACHE_DIR. Each keeps answers from an
|
||||
// outside service, which are fetched again when the file is lost:
|
||||
// - lookups.json holds GeoJS's answers;
|
||||
// - lists.json holds the last try and last good copy of each list
|
||||
// fetched from a URL;
|
||||
// - dnsbl.json holds the DNSBL zones' verdicts;
|
||||
// - abuseipdb-scores.json holds AbuseIPDB's scores;
|
||||
// - routewatch.json holds routewatch's answers.
|
||||
//
|
||||
// Load reads them at start, Watch takes in an admin's edit of one while
|
||||
// sww runs, and Run and WriteAll write them. The disk is read and
|
||||
// written outside the parts' locks, which are held only to take a
|
||||
// snapshot or to put in what a file holds, so that no request waits on
|
||||
@@ -31,6 +41,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/civil"
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||
@@ -39,6 +50,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||
)
|
||||
|
||||
// version is the version of the files' format, the only one read.
|
||||
@@ -48,15 +60,42 @@ const version = 1
|
||||
// hold visitors' addresses.
|
||||
const fileMode = 0o600
|
||||
|
||||
// The state files' names.
|
||||
// The state files' names, in Dir.
|
||||
const (
|
||||
bansJSON = "bans.json"
|
||||
clientsJSON = "clients.json"
|
||||
lookupsJSON = "lookups.json"
|
||||
reputationJSON = "reputation.json"
|
||||
alertsJSON = "alerts.json"
|
||||
bansJSON = "bans.json"
|
||||
clientsJSON = "clients.json"
|
||||
abuseIPDBChecksSpentJSON = "abuseipdb-checks-spent.json"
|
||||
alertsJSON = "alerts.json"
|
||||
)
|
||||
|
||||
// The cache files' names, in CacheDir.
|
||||
const (
|
||||
lookupsJSON = "lookups.json"
|
||||
listsJSON = "lists.json"
|
||||
dnsblJSON = "dnsbl.json"
|
||||
abuseIPDBScoresJSON = "abuseipdb-scores.json"
|
||||
routeWatchJSON = "routewatch.json"
|
||||
)
|
||||
|
||||
// fileNames returns the names of the state files, then those of the
|
||||
// cache files, in the order they are written.
|
||||
func fileNames() []string {
|
||||
return []string{
|
||||
bansJSON, clientsJSON, abuseIPDBChecksSpentJSON, alertsJSON,
|
||||
lookupsJSON, listsJSON, dnsblJSON, abuseIPDBScoresJSON, routeWatchJSON,
|
||||
}
|
||||
}
|
||||
|
||||
// isCacheFile reports whether name is the name of a cache file.
|
||||
func isCacheFile(name string) bool {
|
||||
switch name {
|
||||
case lookupsJSON, listsJSON, dnsblJSON, abuseIPDBScoresJSON, routeWatchJSON:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
errVersion = errors.New("unknown version")
|
||||
// errMissing is for an entry without a field it needs.
|
||||
@@ -72,24 +111,29 @@ var (
|
||||
|
||||
// Params are what Load needs.
|
||||
type Params struct {
|
||||
// Dir is the directory of the state files (SWW_STATE_DIR).
|
||||
Dir string
|
||||
// Dir is the directory of the state files (SWW_STATE_DIR), and
|
||||
// CacheDir that of the cache files (SWW_CACHE_DIR).
|
||||
Dir string
|
||||
CacheDir string
|
||||
// WriteDelay is how long after a ban is made bans.json is written
|
||||
// (SWW_STATE_WRITE_DELAY), and CounterInterval how often every file
|
||||
// is (SWW_STATE_COUNTER_INTERVAL).
|
||||
WriteDelay time.Duration
|
||||
CounterInterval time.Duration
|
||||
// Ledger, Limiter, GeoJS, Lists, DNSBL, AbuseIPDB, Alerts and Anomalies
|
||||
// hold the state. Alerts also receive a file_error alert for an edit set
|
||||
// aside, and for a write that fails while sww runs.
|
||||
Ledger *bans.Ledger
|
||||
Limiter *ratelimit.Limiter
|
||||
GeoJS *lookup.GeoJS
|
||||
Lists *reputation.Lists
|
||||
DNSBL *reputation.DNSBL
|
||||
AbuseIPDB *reputation.AbuseIPDB
|
||||
Alerts *alerts.Queue
|
||||
Anomalies *anomaly.Counters
|
||||
// Ledger, Limiter, GeoJS, Lists, DNSBL, AbuseIPDB, RouteWatch, Alerts and
|
||||
// Anomalies hold the state. RouteWatch is nil while SWW_ROUTEWATCH_URL is
|
||||
// unset, and routewatch.json then holds no answers. Alerts also receive a
|
||||
// file_error alert for an edit set aside, and for a write that fails while
|
||||
// sww runs.
|
||||
Ledger *bans.Ledger
|
||||
Limiter *ratelimit.Limiter
|
||||
GeoJS *lookup.GeoJS
|
||||
Lists *reputation.Lists
|
||||
DNSBL *reputation.DNSBL
|
||||
AbuseIPDB *reputation.AbuseIPDB
|
||||
RouteWatch *routewatch.Client
|
||||
Alerts *alerts.Queue
|
||||
Anomalies *anomaly.Counters
|
||||
// Now tells the time by which the counters' buckets run out, normally
|
||||
// time.Now in UTC.
|
||||
Now func() time.Time
|
||||
@@ -101,7 +145,7 @@ type Params struct {
|
||||
Metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
// Files are the state files of a running sww.
|
||||
// Files are the state and cache files of a running sww.
|
||||
type Files struct {
|
||||
params Params
|
||||
|
||||
@@ -147,13 +191,38 @@ type lookupsFile struct {
|
||||
Lookups []lookup.Answer `json:"lookups"`
|
||||
}
|
||||
|
||||
// reputationFile is reputation.json, indented for an admin to read and
|
||||
// edit, so that each line of a list's copy is on a line of its own.
|
||||
type reputationFile struct {
|
||||
Version int `json:"version"`
|
||||
Lists []reputation.List `json:"lists"`
|
||||
Verdicts []reputation.Verdict `json:"verdicts"`
|
||||
AbuseIPDB reputation.Checks `json:"abuseipdb"`
|
||||
// abuseIPDBChecksSpentFile is abuseipdb-checks-spent.json, indented for an
|
||||
// admin to read and edit.
|
||||
type abuseIPDBChecksSpentFile struct {
|
||||
Version int `json:"version"`
|
||||
Day civil.Date `json:"day,omitzero"`
|
||||
Spent int `json:"spent"`
|
||||
}
|
||||
|
||||
// listsFile is lists.json, indented for an admin to read and edit, so
|
||||
// that each line of a list's copy is on a line of its own.
|
||||
type listsFile struct {
|
||||
Version int `json:"version"`
|
||||
Lists []reputation.List `json:"lists"`
|
||||
}
|
||||
|
||||
// dnsblFile is dnsbl.json, with each verdict on a line of its own.
|
||||
type dnsblFile struct {
|
||||
Version int `json:"version"`
|
||||
Verdicts []reputation.Verdict `json:"verdicts"`
|
||||
}
|
||||
|
||||
// abuseIPDBScoresFile is abuseipdb-scores.json, with each score on a line
|
||||
// of its own.
|
||||
type abuseIPDBScoresFile struct {
|
||||
Version int `json:"version"`
|
||||
Scores []reputation.Score `json:"scores"`
|
||||
}
|
||||
|
||||
// routeWatchFile is routewatch.json, with each answer on a line of its own.
|
||||
type routeWatchFile struct {
|
||||
Version int `json:"version"`
|
||||
Answers []routewatch.Answer `json:"answers"`
|
||||
}
|
||||
|
||||
// alertsFile is alerts.json, indented for an admin to read and edit.
|
||||
@@ -178,34 +247,45 @@ type stateFile interface {
|
||||
toUTC()
|
||||
}
|
||||
|
||||
// Load checks that files can be written in Dir, and reads the state files
|
||||
// in it into the parts of Params that hold the state. A missing file is
|
||||
// empty state, as on a first start. A file that does not parse, has an
|
||||
// unknown version, or has an entry without a field it needs, is an error
|
||||
// that names the file and, where the JSON decoder tells it, the line and
|
||||
// column, or else the entry.
|
||||
// Load checks that files can be written in Dir and CacheDir, and reads the
|
||||
// files in them into the parts of Params that hold the state. A missing
|
||||
// file is empty state, as on a first start. A state file that does not
|
||||
// parse, has an unknown version, or has an entry without a field it
|
||||
// needs, is an error that names the file and, where the JSON decoder
|
||||
// tells it, the line and column, or else the entry. A cache file is never
|
||||
// an error, as readCache describes.
|
||||
func Load(params Params) (*Files, error) {
|
||||
err := checkWritable(params.Dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("SWW_STATE_DIR cannot be written: %w", err)
|
||||
}
|
||||
|
||||
err = checkWritable(params.CacheDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("SWW_CACHE_DIR cannot be written: %w", err)
|
||||
}
|
||||
|
||||
f := &Files{params: params, sums: map[string][sha256.Size]byte{}}
|
||||
|
||||
bansRead, bansErr := f.read(bansJSON)
|
||||
clientsRead, clientsErr := f.read(clientsJSON)
|
||||
lookupsRead, lookupsErr := f.read(lookupsJSON)
|
||||
reputationRead, reputationErr := f.read(reputationJSON)
|
||||
_, checksErr := f.read(abuseIPDBChecksSpentJSON)
|
||||
alertsRead, alertsErr := f.read(alertsJSON)
|
||||
|
||||
err = errors.Join(bansErr, clientsErr, lookupsErr, reputationErr, alertsErr)
|
||||
err = errors.Join(bansErr, clientsErr, checksErr, alertsErr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
params.ProcessLog.Info("read the state files", "directory", params.Dir,
|
||||
"bans", bansRead, "clients", clientsRead, "lookups", lookupsRead,
|
||||
"lists", reputationRead, "alerts_waiting", alertsRead)
|
||||
lookupsRead := f.readCache(lookupsJSON)
|
||||
listsRead := f.readCache(listsJSON)
|
||||
f.readCache(dnsblJSON)
|
||||
f.readCache(abuseIPDBScoresJSON)
|
||||
f.readCache(routeWatchJSON)
|
||||
|
||||
params.ProcessLog.Info("read the state and cache files", "directory", params.Dir,
|
||||
"cache_directory", params.CacheDir, "bans", bansRead, "clients", clientsRead,
|
||||
"lookups", lookupsRead, "lists", listsRead, "alerts_waiting", alertsRead)
|
||||
|
||||
return f, nil
|
||||
}
|
||||
@@ -234,69 +314,101 @@ func (f *Files) Run(ctx context.Context) {
|
||||
|
||||
f.logFailure(bansJSON, f.writeFile(bansJSON))
|
||||
case <-interval.C:
|
||||
for _, name := range []string{
|
||||
bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
|
||||
} {
|
||||
for _, name := range fileNames() {
|
||||
f.logFailure(name, f.writeFile(name))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WriteAll writes every state file, as sww stops. A file that
|
||||
// WriteAll writes every state and cache file, as sww stops. A file that
|
||||
// fails does not keep the others from being written.
|
||||
func (f *Files) WriteAll() error {
|
||||
return errors.Join(f.writeFile(bansJSON), f.writeFile(clientsJSON),
|
||||
f.writeFile(lookupsJSON), f.writeFile(reputationJSON), f.writeFile(alertsJSON))
|
||||
}
|
||||
names := fileNames()
|
||||
errs := make([]error, 0, len(names))
|
||||
|
||||
// Watch watches Dir until ctx is done, and takes in an admin's edit of a
|
||||
// state file as soon as it is saved: what the file holds replaces what
|
||||
// sww held for it. An edit that does not parse is left for the
|
||||
// file's next write, which sets it aside, since a file can be read while
|
||||
// an editor is still writing it. If Dir cannot be watched, that is
|
||||
// logged, and an edit is taken in only before its file is written.
|
||||
func (f *Files) Watch(ctx context.Context) {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err == nil {
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
err = watcher.Add(f.params.Dir)
|
||||
for _, name := range names {
|
||||
errs = append(errs, f.writeFile(name))
|
||||
}
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// Watch watches Dir and CacheDir until ctx is done, and takes in an
|
||||
// admin's edit of a file as soon as it is saved: what the file holds
|
||||
// replaces what sww held for it. An edit that does not parse is left for
|
||||
// the file's next write, which sets it aside, since a file can be read
|
||||
// while an editor is still writing it. A directory that cannot be watched
|
||||
// is logged, and the other is still watched. An edit in a directory not
|
||||
// watched is taken in only before its file is written.
|
||||
func (f *Files) Watch(ctx context.Context) {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
f.params.ProcessLog.Error("cannot watch the state files for edits",
|
||||
f.params.ProcessLog.Error("cannot watch the state and cache files for edits",
|
||||
"error", err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
f.params.ProcessLog.Info("watching the state files for edits",
|
||||
"directory", f.params.Dir)
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
var watched []string
|
||||
|
||||
for _, dir := range []struct{ path, failed string }{
|
||||
{f.params.Dir, "cannot watch the state files for edits"},
|
||||
{f.params.CacheDir, "cannot watch the cache files for edits"},
|
||||
} {
|
||||
err = watcher.Add(dir.path)
|
||||
if err != nil {
|
||||
f.params.ProcessLog.Error(dir.failed, "directory", dir.path,
|
||||
"error", err.Error())
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
watched = append(watched, dir.path)
|
||||
}
|
||||
|
||||
watching := "watching the state and cache files for edits"
|
||||
|
||||
switch {
|
||||
case len(watched) == 0:
|
||||
return
|
||||
case len(watched) == 1 && watched[0] == f.params.Dir:
|
||||
watching = "watching the state files for edits"
|
||||
case len(watched) == 1:
|
||||
watching = "watching the cache files for edits"
|
||||
}
|
||||
|
||||
f.params.ProcessLog.Info(watching, "directories", watched)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case event := <-watcher.Events:
|
||||
switch name := filepath.Base(event.Name); name {
|
||||
case bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON:
|
||||
name := filepath.Base(event.Name)
|
||||
if slices.Contains(fileNames(), name) &&
|
||||
filepath.Join(f.dir(name), name) == event.Name {
|
||||
f.fileChanged(name)
|
||||
}
|
||||
case err = <-watcher.Errors:
|
||||
f.params.ProcessLog.Warn("watching the state files failed",
|
||||
f.params.ProcessLog.Warn("watching the state and cache files failed",
|
||||
"error", err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// logFailure logs a write of the state file name that failed, and raises
|
||||
// logFailure logs a write of the file name that failed, and raises
|
||||
// a file_error alert for it.
|
||||
func (f *Files) logFailure(name string, err error) {
|
||||
if err != nil {
|
||||
const failed = "writing the state files failed"
|
||||
failed := "writing the state files failed"
|
||||
if isCacheFile(name) {
|
||||
failed = "writing the cache files failed"
|
||||
}
|
||||
|
||||
// Raised before it is logged, so that the alert is there once the
|
||||
// log line is.
|
||||
@@ -304,14 +416,14 @@ func (f *Files) logFailure(name string, err error) {
|
||||
Event: alerts.EventFileError,
|
||||
Reason: failed,
|
||||
Detail: map[string]any{
|
||||
"file": filepath.Join(f.params.Dir, name), "error": err.Error(),
|
||||
"file": filepath.Join(f.dir(name), name), "error": err.Error(),
|
||||
},
|
||||
})
|
||||
f.params.ProcessLog.Error(failed, "error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// fileChanged takes in what the state file name holds, as Watch sees it
|
||||
// fileChanged takes in what the file name holds, as Watch sees it
|
||||
// change, if that is an edit made since sww last read or wrote
|
||||
// the file. A file that cannot be read or does not parse is left for its
|
||||
// next write.
|
||||
@@ -327,7 +439,7 @@ func (f *Files) fileChanged(name string) {
|
||||
_ = f.takeInEdit(name, data)
|
||||
}
|
||||
|
||||
// takeInEdit takes in data, an edit of the state file name, as takeIn
|
||||
// takeInEdit takes in data, an edit of the file name, as takeIn
|
||||
// does, and counts and logs it. Every edit taken in while sww
|
||||
// runs, by Watch or by a write, is taken in here. An edit that does not
|
||||
// parse is neither counted nor logged, and takeIn's error returned.
|
||||
@@ -337,17 +449,31 @@ func (f *Files) takeInEdit(name string, data []byte) error {
|
||||
return err
|
||||
}
|
||||
|
||||
tookIn := "took in an edit of a state file"
|
||||
if isCacheFile(name) {
|
||||
tookIn = "took in an edit of a cache file"
|
||||
}
|
||||
|
||||
// Counted before it is logged, so that the count is there once the
|
||||
// log line is.
|
||||
f.params.Metrics.StateFileEditTakenIn(name)
|
||||
f.params.ProcessLog.Info("took in an edit of a state file",
|
||||
"file", filepath.Join(f.params.Dir, name))
|
||||
f.params.ProcessLog.Info(tookIn, "file", filepath.Join(f.dir(name), name))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// read takes in the state file name at start, and returns how many
|
||||
// entries it holds. A missing file holds none.
|
||||
// dir returns the directory of the file name: CacheDir for a cache file,
|
||||
// and Dir for a state file.
|
||||
func (f *Files) dir(name string) string {
|
||||
if isCacheFile(name) {
|
||||
return f.params.CacheDir
|
||||
}
|
||||
|
||||
return f.params.Dir
|
||||
}
|
||||
|
||||
// read takes in the file name at start, and returns how many entries it
|
||||
// holds. A missing file holds none.
|
||||
func (f *Files) read(name string) (int, error) {
|
||||
data, changed, err := f.readChanged(name)
|
||||
if err != nil || !changed {
|
||||
@@ -357,14 +483,44 @@ func (f *Files) read(name string) (int, error) {
|
||||
return f.takeIn(name, data, false)
|
||||
}
|
||||
|
||||
// readChanged returns what the state file name holds, and whether that
|
||||
// has changed since sww last read or wrote the file, as it has
|
||||
// for a file sww never read or wrote. A missing file has not
|
||||
// changed: it is written again at its next write.
|
||||
func (f *Files) readChanged(name string) ([]byte, bool, error) {
|
||||
path := filepath.Join(f.params.Dir, name)
|
||||
// readCache takes in the cache file name at start, as read does. A cache
|
||||
// file that cannot be read, or does not parse, does not stop the start:
|
||||
// it is logged, and its cache starts empty and fills again. One that does
|
||||
// not parse is set aside as an edit is, and one that cannot be read is
|
||||
// written over at its next write.
|
||||
func (f *Files) readCache(name string) int {
|
||||
data, changed, err := f.readChanged(name)
|
||||
if err != nil || !changed {
|
||||
f.logUnread(name, err)
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // a state file, in SWW_STATE_DIR
|
||||
return 0
|
||||
}
|
||||
|
||||
entries, err := f.takeIn(name, data, false)
|
||||
if err != nil {
|
||||
f.logUnread(name, f.setAside(name, err))
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
// logUnread logs err, the reason the cache file name was not read at
|
||||
// start, unless it is nil.
|
||||
func (f *Files) logUnread(name string, err error) {
|
||||
if err != nil {
|
||||
f.params.ProcessLog.Warn("cannot read a cache file, so its cache starts empty",
|
||||
"file", filepath.Join(f.dir(name), name), "error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// readChanged returns what the file name holds, and whether that has
|
||||
// changed since sww last read or wrote the file, as it has for a file
|
||||
// sww never read or wrote. A missing file has not changed: it is written
|
||||
// again at its next write.
|
||||
func (f *Files) readChanged(name string) ([]byte, bool, error) {
|
||||
path := filepath.Join(f.dir(name), name)
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // a file in one of the two directories
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, false, nil
|
||||
}
|
||||
@@ -376,15 +532,39 @@ func (f *Files) readChanged(name string) ([]byte, bool, error) {
|
||||
return data, sha256.Sum256(data) != f.sums[name], nil
|
||||
}
|
||||
|
||||
// takeIn parses data, what the state file name holds, puts it into the
|
||||
// takeIn parses data, what the file name holds, puts it into the
|
||||
// part that keeps that state, in place of what the part held, and returns
|
||||
// how many entries the file holds. edit is whether data is an admin's
|
||||
// edit taken in while sww runs, rather than the file read at the
|
||||
// start. An error names the file and, where the JSON decoder tells it,
|
||||
// the line and column, or else the entry.
|
||||
// the line and column, or else the entry. abuseipdb-checks-spent.json
|
||||
// holds no entries.
|
||||
func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
|
||||
path := filepath.Join(f.params.Dir, name)
|
||||
path := filepath.Join(f.dir(name), name)
|
||||
|
||||
var (
|
||||
entries int
|
||||
err error
|
||||
)
|
||||
|
||||
if isCacheFile(name) {
|
||||
entries, err = f.takeInCache(name, path, data)
|
||||
} else {
|
||||
entries, err = f.takeInState(name, path, data, edit)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
f.sums[name] = sha256.Sum256(data)
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// takeInState parses data, what the state file name, at path, holds, and
|
||||
// puts it in, as takeIn describes.
|
||||
func (f *Files) takeInState(name, path string, data []byte, edit bool) (int, error) {
|
||||
var entries int
|
||||
|
||||
switch name {
|
||||
@@ -418,6 +598,33 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
|
||||
|
||||
f.params.Limiter.Load(file.Clients, f.params.Now())
|
||||
entries = len(file.Clients)
|
||||
case abuseIPDBChecksSpentJSON:
|
||||
var file abuseIPDBChecksSpentFile
|
||||
|
||||
err := parse(path, data, &file)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
f.params.AbuseIPDB.LoadChecksSpent(reputation.ChecksSpent{
|
||||
Day: file.Day, Spent: file.Spent,
|
||||
})
|
||||
case alertsJSON:
|
||||
waiting, err := f.takeInAlerts(path, data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
entries = waiting
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// takeInCache parses data, what the cache file name, at path, holds, and
|
||||
// puts it in, as takeIn describes.
|
||||
func (f *Files) takeInCache(name, path string, data []byte) (int, error) {
|
||||
switch name {
|
||||
case lookupsJSON:
|
||||
var file lookupsFile
|
||||
|
||||
@@ -427,9 +634,10 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
|
||||
}
|
||||
|
||||
f.params.GeoJS.Load(file.Lookups)
|
||||
entries = len(file.Lookups)
|
||||
case reputationJSON:
|
||||
var file reputationFile
|
||||
|
||||
return len(file.Lookups), nil
|
||||
case listsJSON:
|
||||
var file listsFile
|
||||
|
||||
err := parse(path, data, &file)
|
||||
if err != nil {
|
||||
@@ -441,21 +649,43 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
|
||||
return 0, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
|
||||
f.params.DNSBL.Load(file.Verdicts)
|
||||
f.params.AbuseIPDB.Load(file.AbuseIPDB)
|
||||
entries = len(file.Lists)
|
||||
case alertsJSON:
|
||||
waiting, err := f.takeInAlerts(path, data)
|
||||
return len(file.Lists), nil
|
||||
case dnsblJSON:
|
||||
var file dnsblFile
|
||||
|
||||
err := parse(path, data, &file)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
entries = waiting
|
||||
f.params.DNSBL.Load(file.Verdicts)
|
||||
|
||||
return len(file.Verdicts), nil
|
||||
case routeWatchJSON:
|
||||
var file routeWatchFile
|
||||
|
||||
err := parse(path, data, &file)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if f.params.RouteWatch != nil {
|
||||
f.params.RouteWatch.Load(file.Answers)
|
||||
}
|
||||
|
||||
return len(file.Answers), nil
|
||||
default: // abuseipdb-scores.json
|
||||
var file abuseIPDBScoresFile
|
||||
|
||||
err := parse(path, data, &file)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
f.params.AbuseIPDB.Load(file.Scores)
|
||||
|
||||
return len(file.Scores), nil
|
||||
}
|
||||
|
||||
f.sums[name] = sha256.Sum256(data)
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// takeInAlerts parses data, what alerts.json, at path, holds, puts it
|
||||
@@ -493,17 +723,22 @@ func (f *Files) takeInAlerts(path string, data []byte) (int, error) {
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// writeFile writes the state file name from what sww holds. An
|
||||
// edit made since sww last read or wrote the file is taken in
|
||||
// first, so that it is not overwritten, or set aside if it does not
|
||||
// parse. A file that cannot be read, or an edit that cannot be set
|
||||
// aside, is left as it is, and the write given up. Every write is counted
|
||||
// in the metrics, and one that fails or is given up as a failure.
|
||||
// writeFile writes the file name from what sww holds. An edit made since
|
||||
// sww last read or wrote the file is taken in first, so that it is not
|
||||
// overwritten, or set aside if it does not parse. A state file that
|
||||
// cannot be read, or an edit that cannot be set aside, is left as it is,
|
||||
// and the write given up. A cache file that cannot be read is written
|
||||
// over, since what it held is fetched again. Every write is counted in
|
||||
// the metrics, and one that fails or is given up as a failure.
|
||||
func (f *Files) writeFile(name string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
data, changed, err := f.readChanged(name)
|
||||
if err != nil && isCacheFile(name) {
|
||||
err = nil
|
||||
}
|
||||
|
||||
if err == nil && changed {
|
||||
err = f.takeInEdit(name, data)
|
||||
if err != nil {
|
||||
@@ -519,7 +754,7 @@ func (f *Files) writeFile(name string) error {
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
err = write(f.params.Dir, name, data)
|
||||
err = write(f.dir(name), name, data)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
@@ -527,7 +762,7 @@ func (f *Files) writeFile(name string) error {
|
||||
// fails, so that its next read does not take it for an admin's
|
||||
// edit.
|
||||
f.sums[name] = sha256.Sum256(data)
|
||||
err = syncDirectory(f.params.Dir)
|
||||
err = syncDirectory(f.dir(name))
|
||||
}
|
||||
|
||||
f.params.Metrics.StateFileWritten(name, len(data), err)
|
||||
@@ -535,20 +770,23 @@ func (f *Files) writeFile(name string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// setAside renames the state file name, an edit that does not parse with
|
||||
// setAside renames the file name, an edit that does not parse with
|
||||
// parseErr, to name.bad, for the admin to mend, and logs it with where in
|
||||
// the file the error is, and raises a file_error alert for it. If the
|
||||
// rename fails, the edit is left as it is, and the error returned is
|
||||
// parseErr joined with the rename's.
|
||||
func (f *Files) setAside(name string, parseErr error) error {
|
||||
path := filepath.Join(f.params.Dir, name)
|
||||
path := filepath.Join(f.dir(name), name)
|
||||
|
||||
err := os.Rename(path, path+".bad")
|
||||
if err != nil {
|
||||
return errors.Join(parseErr, err)
|
||||
}
|
||||
|
||||
const setAside = "set aside an edit of a state file that does not parse"
|
||||
setAside := "set aside an edit of a state file that does not parse"
|
||||
if isCacheFile(name) {
|
||||
setAside = "set aside an edit of a cache file that does not parse"
|
||||
}
|
||||
|
||||
// Raised before it is logged, so that the alert is there once the log
|
||||
// line is.
|
||||
@@ -563,7 +801,7 @@ func (f *Files) setAside(name string, parseErr error) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// encode returns the state file name as sww writes it, from a
|
||||
// encode returns the file name as sww writes it, from a
|
||||
// snapshot of the part that keeps that state.
|
||||
func (f *Files) encode(name string) ([]byte, error) {
|
||||
switch name {
|
||||
@@ -575,10 +813,24 @@ func (f *Files) encode(name string) ([]byte, error) {
|
||||
return encodeOnePerLine("clients", f.params.Limiter.Snapshot())
|
||||
case lookupsJSON:
|
||||
return encodeOnePerLine("lookups", f.params.GeoJS.Snapshot())
|
||||
case reputationJSON:
|
||||
return encodeIndented(reputationFile{
|
||||
Version: version, Lists: f.params.Lists.Snapshot(),
|
||||
Verdicts: f.params.DNSBL.Snapshot(), AbuseIPDB: f.params.AbuseIPDB.Snapshot(),
|
||||
case listsJSON:
|
||||
return encodeIndented(listsFile{Version: version, Lists: f.params.Lists.Snapshot()})
|
||||
case dnsblJSON:
|
||||
return encodeOnePerLine("verdicts", f.params.DNSBL.Snapshot())
|
||||
case abuseIPDBScoresJSON:
|
||||
return encodeOnePerLine("scores", f.params.AbuseIPDB.Snapshot())
|
||||
case routeWatchJSON:
|
||||
var answers []routewatch.Answer
|
||||
if f.params.RouteWatch != nil {
|
||||
answers = f.params.RouteWatch.Snapshot()
|
||||
}
|
||||
|
||||
return encodeOnePerLine("answers", answers)
|
||||
case abuseIPDBChecksSpentJSON:
|
||||
checks := f.params.AbuseIPDB.ChecksSpent()
|
||||
|
||||
return encodeIndented(abuseIPDBChecksSpentFile{
|
||||
Version: version, Day: checks.Day, Spent: checks.Spent,
|
||||
})
|
||||
default: // alerts.json
|
||||
held := f.params.Alerts.Snapshot()
|
||||
@@ -748,14 +1000,8 @@ func (f *lookupsFile) check(data []byte) error {
|
||||
// check refuses a list without its URL, which would name no list, or the
|
||||
// time it was last tried, which would have it fetched at once, and a copy
|
||||
// of it without the time it was fetched, or without its lines, which hold
|
||||
// the list. It refuses a verdict without its zone or its client, which
|
||||
// would be about no one, whether the zone lists the client, or the time
|
||||
// it was fetched, which would drop it, and so an AbuseIPDB score without
|
||||
// its client, the score, or the time it was fetched. A verdict's listed is
|
||||
// false for a client the zone does not list, and a score can be 0, which
|
||||
// the structs cannot tell from a missing one, so each is read again as
|
||||
// written.
|
||||
func (f *reputationFile) check(data []byte) error {
|
||||
// the list.
|
||||
func (f *listsFile) check([]byte) error {
|
||||
for i, kept := range f.Lists {
|
||||
switch {
|
||||
case kept.URL == "":
|
||||
@@ -769,6 +1015,15 @@ func (f *reputationFile) check(data []byte) error {
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// check refuses a verdict without its zone or its client, which would be
|
||||
// about no one, whether the zone lists the client, or the time it was
|
||||
// fetched, which would drop it. A verdict's listed is false for a client
|
||||
// the zone does not list, which Verdict cannot tell from a missing one,
|
||||
// so each listed is read again as written.
|
||||
func (f *dnsblFile) check(data []byte) error {
|
||||
var written struct {
|
||||
Verdicts []struct {
|
||||
Listed *bool `json:"listed"`
|
||||
@@ -783,28 +1038,27 @@ func (f *reputationFile) check(data []byte) error {
|
||||
for i, verdict := range f.Verdicts {
|
||||
switch {
|
||||
case verdict.Zone == "":
|
||||
return fmt.Errorf("verdicts %w", missing(i, "zone"))
|
||||
return missing(i, "zone")
|
||||
case !verdict.Client.IsValid():
|
||||
return fmt.Errorf("verdicts %w", missing(i, "client"))
|
||||
return missing(i, "client")
|
||||
case written.Verdicts[i].Listed == nil:
|
||||
return fmt.Errorf("verdicts %w", missing(i, "listed"))
|
||||
return missing(i, "listed")
|
||||
case verdict.Fetched.IsZero():
|
||||
return fmt.Errorf("verdicts %w", missing(i, "fetched"))
|
||||
return missing(i, "fetched")
|
||||
}
|
||||
}
|
||||
|
||||
return checkScores(f.AbuseIPDB.Scores, data)
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkScores refuses an AbuseIPDB score, of scores, read from data, as
|
||||
// reputationFile's check describes.
|
||||
func checkScores(scores []reputation.Score, data []byte) error {
|
||||
// check refuses an AbuseIPDB score without its client, the score, or the
|
||||
// time it was fetched. A score can be 0, which Score cannot tell from a
|
||||
// missing one, so each score is read again as written.
|
||||
func (f *abuseIPDBScoresFile) check(data []byte) error {
|
||||
var written struct {
|
||||
AbuseIPDB struct {
|
||||
Scores []struct {
|
||||
Score *int64 `json:"score"`
|
||||
} `json:"scores"`
|
||||
} `json:"abuseipdb"`
|
||||
Scores []struct {
|
||||
Score *int64 `json:"score"`
|
||||
} `json:"scores"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal(data, &written)
|
||||
@@ -812,20 +1066,57 @@ func checkScores(scores []reputation.Score, data []byte) error {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, kept := range scores {
|
||||
for i, kept := range f.Scores {
|
||||
switch {
|
||||
case !kept.Client.IsValid():
|
||||
return fmt.Errorf("abuseipdb scores %w", missing(i, "client"))
|
||||
case written.AbuseIPDB.Scores[i].Score == nil:
|
||||
return fmt.Errorf("abuseipdb scores %w", missing(i, "score"))
|
||||
return missing(i, "client")
|
||||
case written.Scores[i].Score == nil:
|
||||
return missing(i, "score")
|
||||
case kept.Fetched.IsZero():
|
||||
return fmt.Errorf("abuseipdb scores %w", missing(i, "fetched"))
|
||||
return missing(i, "fetched")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// check refuses an answer without its client, which would answer for
|
||||
// nobody, its netblock, or the time routewatch gave it, which would drop
|
||||
// it. "" is the netblock of an address without a live route, which Answer
|
||||
// cannot tell from a missing one, so each netblock is read again as
|
||||
// written.
|
||||
func (f *routeWatchFile) check(data []byte) error {
|
||||
var written struct {
|
||||
Answers []struct {
|
||||
Netblock *string `json:"netblock"`
|
||||
} `json:"answers"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal(data, &written)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, answer := range f.Answers {
|
||||
switch {
|
||||
case !answer.Client.IsValid():
|
||||
return missing(i, "client")
|
||||
case written.Answers[i].Netblock == nil:
|
||||
return missing(i, "netblock")
|
||||
case answer.Answered.IsZero():
|
||||
return missing(i, "answered")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// check refuses nothing, since a missing day or count means no check
|
||||
// spent.
|
||||
func (f *abuseIPDBChecksSpentFile) check([]byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// check refuses a cooldown without its event or when its alert was sent,
|
||||
// which would hold back no repeat, alerts waiting for a destination with
|
||||
// another name than webhook, slack or ntfy, most likely misspelt, an
|
||||
@@ -947,22 +1238,37 @@ func (f *lookupsFile) toUTC() {
|
||||
}
|
||||
}
|
||||
|
||||
// toUTC converts every time of each list, verdict and AbuseIPDB score to
|
||||
// UTC.
|
||||
func (f *reputationFile) toUTC() {
|
||||
// toUTC converts both times of each list to UTC.
|
||||
func (f *listsFile) toUTC() {
|
||||
for i := range f.Lists {
|
||||
convertToUTC(&f.Lists[i].Tried, &f.Lists[i].Fetched)
|
||||
}
|
||||
}
|
||||
|
||||
// toUTC converts the time of each verdict to UTC.
|
||||
func (f *dnsblFile) toUTC() {
|
||||
for i := range f.Verdicts {
|
||||
convertToUTC(&f.Verdicts[i].Fetched)
|
||||
}
|
||||
}
|
||||
|
||||
for i := range f.AbuseIPDB.Scores {
|
||||
convertToUTC(&f.AbuseIPDB.Scores[i].Fetched)
|
||||
// toUTC converts the time of each AbuseIPDB score to UTC.
|
||||
func (f *abuseIPDBScoresFile) toUTC() {
|
||||
for i := range f.Scores {
|
||||
convertToUTC(&f.Scores[i].Fetched)
|
||||
}
|
||||
}
|
||||
|
||||
// toUTC converts the time of each answer to UTC.
|
||||
func (f *routeWatchFile) toUTC() {
|
||||
for i := range f.Answers {
|
||||
convertToUTC(&f.Answers[i].Answered)
|
||||
}
|
||||
}
|
||||
|
||||
// toUTC does nothing: abuseipdb-checks-spent.json holds a day, not a time.
|
||||
func (f *abuseIPDBChecksSpentFile) toUTC() {}
|
||||
|
||||
// toUTC converts every time of each cooldown, of the hour, of each alert
|
||||
// waiting and of each anomaly counter to UTC.
|
||||
func (f *alertsFile) toUTC() {
|
||||
@@ -1114,7 +1420,7 @@ func write(dir, name string, data []byte) error {
|
||||
|
||||
// syncDirectory syncs dir to the disk, so that a rename in it lasts.
|
||||
func syncDirectory(dir string) error {
|
||||
directory, err := os.Open(dir) //nolint:gosec // SWW_STATE_DIR itself
|
||||
directory, err := os.Open(dir) //nolint:gosec // SWW_STATE_DIR or SWW_CACHE_DIR itself
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1124,7 +1430,7 @@ func syncDirectory(dir string) error {
|
||||
|
||||
// writeSynced writes data to the file at path, and syncs it to the disk.
|
||||
func writeSynced(path string, data []byte) error {
|
||||
//nolint:gosec // a state file's temporary file, in SWW_STATE_DIR
|
||||
//nolint:gosec // a file's temporary file, in SWW_STATE_DIR or SWW_CACHE_DIR
|
||||
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileMode)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
Reference in new issue
Block a user