Compare commits

..
1 Commits
Author SHA1 Message Date
sneak 965936ab4d Caches of outside APIs live in /var/cache/sww (closes #314)
check / check (push) Successful in 4m32s
The answers of GeoJS, the lists fetched by URL, the DNS blocklists,
AbuseIPDB and routewatch each go in a cache file in SWW_CACHE_DIR
(default /var/cache/sww): lookups.json, lists.json, dnsbl.json,
abuseipdb-scores.json and routewatch.json. reputation.json is gone. The
AbuseIPDB checks spent today are a count, not a cache, and stay in
SWW_STATE_DIR as abuseipdb-checks-spent.json.

A cache file never stops the start: one that does not parse is set
aside as an edit is, one that cannot be read is written over, and its
cache starts empty. An unwritable SWW_CACHE_DIR stops the start, as an
unwritable SWW_STATE_DIR does. Each directory is watched on its own.
The run script makes the cache directory and gives it to sww.

Model: opus-5-5
2026-11-11 07:23:39 +00:00
7 changed files with 36 additions and 42 deletions

No files matched your search

+1 -2
View File
@@ -2353,8 +2353,7 @@ but for these headers:
claims and are replaced. `X-Forwarded-For` names the peer, `X-Forwarded-Host` claims and are replaced. `X-Forwarded-For` names the peer, `X-Forwarded-Host`
the host it asked for, and `X-Forwarded-Proto` plain `http`. Every other the host it asked for, and `X-Forwarded-Proto` plain `http`. Every other
header whose name starts with `X-Forwarded-` is removed, and so are header whose name starts with `X-Forwarded-` is removed, and so are
`Forwarded` and `X-Real-IP`. A name spelt with `_` for `-`, such as `Forwarded` and `X-Real-IP`.
`X_Forwarded_Ssl`, counts as the same name.
- `X-Request-ID`, which carries the fresh id. - `X-Request-ID`, which carries the fresh id.
- `X-Client-ASN` and `X-Client-Country`, which carry what `waf` answered. A - `X-Client-ASN` and `X-Client-Country`, which carry what `waf` answered. A
header it did not give is left out. Any of the two the visitor sent is header it did not give is left out. Any of the two the visitor sent is
+3 -5
View File
@@ -56,11 +56,9 @@ const (
// of a list failing, a query to a DNSBL zone or a check with AbuseIPDB // of a list failing, a query to a DNSBL zone or a check with AbuseIPDB
// failing or refused, or the day's AbuseIPDB checks used up. // failing or refused, or the day's AbuseIPDB checks used up.
EventSourceFailure = "source_failure" EventSourceFailure = "source_failure"
// EventFileError is one of: // EventFileError is a rule file or state file edited while sww
// - a rule file or state file edited while sww runs that does not parse; // runs that does not parse, a replacement of the lookup database that
// - a cache file that does not parse, at start or after an edit; // cannot be read, or a state file that cannot be written.
// - a state file or cache file that cannot be written;
// - a replacement of the lookup database that cannot be read.
EventFileError = "file_error" EventFileError = "file_error"
// EventSummary is the summary sent as an hour ends: of the alerts held // EventSummary is the summary sent as an hour ends: of the alerts held
// back in it past SWW_ALERT_MAX_PER_HOUR, and of the repeats held // back in it past SWW_ALERT_MAX_PER_HOUR, and of the repeats held
+10 -12
View File
@@ -120,19 +120,17 @@ func New(topN int, instanceName string) *Metrics {
"went on without it, because GeoJS had not given it in time.", "went on without it, because GeoJS had not given it in time.",
}), }),
stateFileWrites: counterVec("sww_state_file_writes_total", stateFileWrites: counterVec("sww_state_file_writes_total",
"Writes of each state or cache file.", byFile), "Writes of each state file.", byFile),
stateFileWriteFailures: counterVec("sww_state_file_write_failures_total", stateFileWriteFailures: counterVec("sww_state_file_write_failures_total",
"Writes of each state or cache file that failed.", byFile), "Writes of each state file that failed.", byFile),
stateFileLastWrite: gaugeVec("sww_state_file_last_write_timestamp_seconds", stateFileLastWrite: gaugeVec("sww_state_file_last_write_timestamp_seconds",
"When each state or cache file was last written, in seconds since 1970.", "When each state file was last written, in seconds since 1970.", byFile),
byFile),
stateFileSize: gaugeVec("sww_state_file_size_bytes", stateFileSize: gaugeVec("sww_state_file_size_bytes",
"The size of each state or cache file, as it was last written.", byFile), "The size of each state file, as it was last written.", byFile),
stateFileEditsTakenIn: counterVec("sww_state_file_edits_taken_in_total", stateFileEditsTakenIn: counterVec("sww_state_file_edits_taken_in_total",
"Edits of each state or cache file taken in while running.", byFile), "Edits of each state file taken in while running.", byFile),
stateFileEditsSetAside: counterVec("sww_state_file_edits_set_aside_total", stateFileEditsSetAside: counterVec("sww_state_file_edits_set_aside_total",
"Edits of each state or cache file renamed to <name>.bad because they "+ "Edits of each state file renamed to <name>.bad because they did not parse.",
"did not parse.",
byFile), byFile),
} }
@@ -449,8 +447,8 @@ func (m *Metrics) CoreRuleSetMatched(mode string, id int) {
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc() m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
} }
// StateFileWritten counts a write of the state or cache file name, of size // StateFileWritten counts a write of the state file name, of size bytes,
// bytes, that ended with err. // that ended with err.
func (m *Metrics) StateFileWritten(name string, size int, err error) { func (m *Metrics) StateFileWritten(name string, size int, err error) {
m.stateFileWrites.WithLabelValues(name).Inc() m.stateFileWrites.WithLabelValues(name).Inc()
@@ -468,13 +466,13 @@ func (m *Metrics) StateFileWritten(name string, size int, err error) {
m.stateFileSize.WithLabelValues(name).Set(float64(size)) m.stateFileSize.WithLabelValues(name).Set(float64(size))
} }
// StateFileEditTakenIn counts an admin's edit of the state or cache file name // StateFileEditTakenIn counts an admin's edit of the state file name
// taken in while sww runs. // taken in while sww runs.
func (m *Metrics) StateFileEditTakenIn(name string) { func (m *Metrics) StateFileEditTakenIn(name string) {
m.stateFileEditsTakenIn.WithLabelValues(name).Inc() m.stateFileEditsTakenIn.WithLabelValues(name).Inc()
} }
// StateFileEditSetAside counts an admin's edit of the state or cache file name // StateFileEditSetAside counts an admin's edit of the state file name
// renamed to name.bad because it did not parse. // renamed to name.bad because it did not parse.
func (m *Metrics) StateFileEditSetAside(name string) { func (m *Metrics) StateFileEditSetAside(name string) {
m.stateFileEditsSetAside.WithLabelValues(name).Inc() m.stateFileEditsSetAside.WithLabelValues(name).Inc()
+6 -5
View File
@@ -105,16 +105,17 @@ func setForwardedHeaders(in, out *http.Request, peer netip.Addr, trusted bool) {
return return
} }
// An app behind CGI or FastCGI reads X_Forwarded_Ssl as X-Forwarded-Ssl, so // ReverseProxy has removed Forwarded and the three set below. Go's server
// a name is matched with each _ read as - and in any letter case. // gives header names in canonical form, so the prefix matches the name in
// any letter case.
for name := range out.Header { for name := range out.Header {
spelt := strings.ToLower(strings.ReplaceAll(name, "_", "-")) if strings.HasPrefix(name, "X-Forwarded-") {
if strings.HasPrefix(spelt, "x-forwarded-") ||
spelt == "forwarded" || spelt == "x-real-ip" {
delete(out.Header, name) delete(out.Header, name)
} }
} }
out.Header.Del("X-Real-IP")
out.Header.Set("X-Forwarded-For", forwardedFor) out.Header.Set("X-Forwarded-For", forwardedFor)
out.Header.Set("X-Forwarded-Host", in.Host) out.Header.Set("X-Forwarded-Host", in.Host)
out.Header.Set("X-Forwarded-Proto", "http") out.Header.Set("X-Forwarded-Proto", "http")
-2
View File
@@ -561,7 +561,6 @@ func TestAPeerNotTrustedHasItsForwardedHeadersReplaced(t *testing.T) {
"X-Forwarded-Tls-Client-Cert-Info", "X-Real-Ip", "X-Forwarded-Tls-Client-Cert-Info", "X-Real-Ip",
"X-Forwarded-Scheme", "X-Forwarded-Ssl", "X-Forwarded-Protocol", "X-Forwarded-Scheme", "X-Forwarded-Ssl", "X-Forwarded-Protocol",
"X-Forwarded-Path", "X-Forwarded-Client-Cert", "X-Forwarded-Path", "X-Forwarded-Client-Cert",
"X_Forwarded_Ssl", "X_Forwarded_Proto", "X_Real_IP",
} }
standIn := startWAF(t, allow) standIn := startWAF(t, allow)
@@ -629,7 +628,6 @@ func TestATrustedProxysForwardedHeadersPassOn(t *testing.T) {
forwardedProtoHeader: "https", forwardedProtoHeader: "https",
"X-Real-Ip": "198.51.100.1", "X-Real-Ip": "198.51.100.1",
"X-Forwarded-Ssl": "on", "X-Forwarded-Ssl": "on",
"X_Forwarded_Ssl": "on",
} }
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
+14 -14
View File
@@ -1,9 +1,9 @@
// Package sww runs the sww process, as the command it is given. // Package sww runs the sww process, as the command it is given.
// `sww waf` reads the settings, the rule files, the lookup database // `sww waf` reads the settings, the rule files, the lookup database
// and the state and cache files, answers rproxy's questions until it is told to // and the state files, answers rproxy's questions until it is told to stop, and
// stop, and then stops in an orderly way, writing the state and cache files. // then stops in an orderly way, writing the state files. `sww rproxy`
// `sww rproxy` reads rproxy's settings alone, and runs the reverse proxy in // reads rproxy's settings alone, and runs the reverse proxy in front of one app
// front of one app that asks waf about each request. `sww healthcheck` is the // that asks waf about each request. `sww healthcheck` is the
// container's health check. // container's health check.
package sww package sww
@@ -101,8 +101,8 @@ func printUsage(w io.Writer) int {
} }
// RunWAF is `sww waf`: it reads the settings, the rule files, // RunWAF is `sww waf`: it reads the settings, the rule files,
// the lookup database and the state and cache files, then serves requests until // the lookup database and the state files, then serves requests until ctx is
// ctx is done. It returns the process's exit status, 1 when waf cannot start. // done. It returns the process's exit status, 1 when waf cannot start.
func RunWAF(ctx context.Context, params Params) int { func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's // Until the settings are read, the one message is an invalid setting's
// error, which every SWW_LOG_LEVEL lets through. // error, which every SWW_LOG_LEVEL lets through.
@@ -171,7 +171,7 @@ func RunWAF(ctx context.Context, params Params) int {
files, err := loadStateFiles(cfg, server, alertQueue, now, processLog) files, err := loadStateFiles(cfg, server, alertQueue, now, processLog)
if err != nil { if err != nil {
processLog.Error("cannot use the state and cache files", "error", err.Error()) processLog.Error("cannot use the state files", "error", err.Error())
return 1 return 1
} }
@@ -236,8 +236,8 @@ func newServer(
return server, nil return server, nil
} }
// loadStateFiles reads the state and cache files into the parts of server and // loadStateFiles reads the state files into the parts of server and into
// into alertQueue, as state.Load does. // alertQueue, as state.Load does.
func loadStateFiles( func loadStateFiles(
cfg *config.Config, server *waf.Server, alertQueue *alerts.Queue, cfg *config.Config, server *waf.Server, alertQueue *alerts.Queue,
now func() time.Time, processLog *slog.Logger, now func() time.Time, processLog *slog.Logger,
@@ -319,13 +319,13 @@ func startSending(
} }
} }
// serve serves requests on listener, writes the state and cache files as they // serve serves requests on listener, writes the state files as they are
// are due, takes in an admin's edits of them, reads the rule files again as // due, takes in an admin's edits of them, reads the rule files again as
// they change, and the lookup database when it is replaced, fetches the // they change, and the lookup database when it is replaced, fetches the
// lists the settings name by URL as they are due, and sends the alerts, // lists the settings name by URL as they are due, and sends the alerts,
// until ctx is done. Then it gives the requests in progress // until ctx is done. Then it gives the requests in progress
// shutdownTimeout to finish, and writes every state and cache file, // shutdownTimeout to finish, and writes every state file, alerts.json with
// alerts.json with the alerts still waiting. // the alerts still waiting.
func serve( func serve(
ctx context.Context, server *waf.Server, listener net.Listener, ctx context.Context, server *waf.Server, listener net.Listener,
files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue, files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue,
@@ -395,7 +395,7 @@ func serve(
err = files.WriteAll() err = files.WriteAll()
if err != nil { if err != nil {
processLog.Error("writing the state and cache files failed", "error", err.Error()) processLog.Error("writing the state files failed", "error", err.Error())
return 1 return 1
} }
+2 -2
View File
@@ -1235,7 +1235,7 @@ func TestUnwritableCacheDirStopsTheStart(t *testing.T) {
t.Fatalf("exit status %d, want 1", status) t.Fatalf("exit status %d, want 1", status)
} }
line := out.line(t, "msg", "cannot use the state and cache files") line := out.line(t, "msg", "cannot use the state files")
if message, _ := line["error"].(string); !strings.HasPrefix(message, if message, _ := line["error"].(string); !strings.HasPrefix(message,
"SWW_CACHE_DIR cannot be written: ") { "SWW_CACHE_DIR cannot be written: ") {
t.Errorf("start refused with %q, want SWW_CACHE_DIR named", message) t.Errorf("start refused with %q, want SWW_CACHE_DIR named", message)
@@ -1260,7 +1260,7 @@ func wantStartRefused(t *testing.T, dir, want string) {
t.Fatalf("exit status %d, want 1", status) t.Fatalf("exit status %d, want 1", status)
} }
line := out.line(t, "msg", "cannot use the state and cache files") line := out.line(t, "msg", "cannot use the state files")
message, _ := line["error"].(string) message, _ := line["error"].(string)
if !strings.HasPrefix(message, want) { if !strings.HasPrefix(message, want) {