Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
249d6a1142 | ||
|
|
0a0850f797 |
No files matched your search
@@ -908,8 +908,6 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||
`internal/alerts`: one check or source each.
|
||||
- `internal/dirwatch`: watches a directory, so that `internal/rules` and
|
||||
`internal/lookup` read their files again when they change.
|
||||
- `internal/state`: the state and cache files.
|
||||
- `internal/requestlog`: the request log.
|
||||
- `internal/processlog`: the process's own messages.
|
||||
|
||||
@@ -69,8 +69,7 @@ never as a time.
|
||||
that command's settings, and runs `internal/rproxy` until `SIGTERM` or
|
||||
`SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check.
|
||||
With no command, or one it does not know, the binary prints the commands
|
||||
and exits with status 2. `sww help` and `sww --help` print them and exit
|
||||
with status 0. `internal/sww` is the only package that imports
|
||||
and exits with status 2. `internal/sww` is the only package that imports
|
||||
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
||||
and stops their background work.
|
||||
- `internal/config`: reads the settings, the one place they are read.
|
||||
@@ -97,9 +96,6 @@ never as a time.
|
||||
imports no other package of this repository: `waf` gives it what it needs
|
||||
to count and report failures.
|
||||
- `internal/lookup`: looks clients up in the lookup database.
|
||||
- `internal/dirwatch`: watches a directory, and calls a function once it has
|
||||
gone a short time without a change. `internal/rules` and `internal/lookup`
|
||||
read their files again this way.
|
||||
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
||||
AbuseIPDB in the background, and keeps their answers. It also fetches
|
||||
AbuseIPDB's blacklist, which `internal/waf` bans.
|
||||
@@ -133,7 +129,6 @@ never as a time.
|
||||
- `github.com/fsnotify/fsnotify` to notice edited files;
|
||||
- `github.com/prometheus/client_golang` for the metrics;
|
||||
- `github.com/spf13/viper` to read the settings from the environment;
|
||||
- `github.com/spf13/cobra` for the commands and their help text;
|
||||
- `github.com/maxmind/mmdbwriter`, in the tests, to write lookup databases;
|
||||
- remote log sending written in this project, since the standard library's
|
||||
`log/syslog` is frozen and writes only the older syslog format.
|
||||
|
||||
@@ -11,7 +11,6 @@ require (
|
||||
github.com/maxmind/mmdbwriter v1.2.0
|
||||
github.com/oschwald/maxminddb-golang/v2 v2.7.0
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
go.uber.org/fx v1.24.0
|
||||
golang.org/x/net v0.60.0
|
||||
@@ -27,7 +26,6 @@ require (
|
||||
github.com/goccy/go-yaml v1.19.2 // indirect
|
||||
github.com/gotnospirit/makeplural v0.0.0-20180622080156-a5f48d94d976 // indirect
|
||||
github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/kaptinlin/go-i18n v0.1.4 // indirect
|
||||
github.com/kaptinlin/jsonschema v0.4.6 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
|
||||
@@ -14,7 +14,6 @@ github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68 h1:KLEjvBBSw
|
||||
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68/go.mod h1:9XocblQEwK6EYylRth7PrPIifF2l0iSoYo4FAXhXU/8=
|
||||
github.com/corazawaf/libinjection-go v0.3.4 h1:jX2Tit2xZxYJJyVryNXucYBZtQqS7473x9Nc6lgRRbk=
|
||||
github.com/corazawaf/libinjection-go v0.3.4/go.mod h1:Ik/+w3UmTWH9yn366RgS9D95K3y7Atb5m/H/gXzzPCk=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0=
|
||||
github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
@@ -35,8 +34,6 @@ github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 h1:c7gcN
|
||||
github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092/go.mod h1:ZZAN4fkkful3l1lpJwF8JbW41ZiG9TwJ2ZlqzQovBNU=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jcchavezs/mergefs v0.1.1 h1:D45R17m6dHnSVZefnhynoeZvcK2Uw0oTrRfoUOQ0S5Y=
|
||||
github.com/jcchavezs/mergefs v0.1.1/go.mod h1:eRLTrsA+vFwQZ48hj8p8gki/5v9C2bFtHH5Mnn4bcGk=
|
||||
github.com/kaptinlin/go-i18n v0.1.4 h1:wCiwAn1LOcvymvWIVAM4m5dUAMiHunTdEubLDk4hTGs=
|
||||
@@ -75,7 +72,6 @@ github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+
|
||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||
@@ -84,9 +80,6 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
@@ -116,7 +109,6 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
|
||||
@@ -135,7 +127,6 @@ golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
|
||||
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
// Package dirwatch watches a directory, and calls a function once the
|
||||
// directory has gone a short time without a change after one. The rule
|
||||
// files and the lookup database are read again this way.
|
||||
package dirwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
)
|
||||
|
||||
// quietTime is how long the directory must go without a change before the
|
||||
// function is called, so that a file still being written, such as one
|
||||
// saved in place, appended to or copied in with scp, is read only once
|
||||
// whole.
|
||||
const quietTime = 2 * time.Second
|
||||
|
||||
// Watcher watches one directory. The directory is watched, rather than a
|
||||
// file in it, so that a file replaced by a rename is seen.
|
||||
type Watcher struct {
|
||||
watcher *fsnotify.Watcher
|
||||
}
|
||||
|
||||
// New starts watching dir, or returns why it cannot.
|
||||
func New(dir string) (*Watcher, error) {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = watcher.Add(dir)
|
||||
if err != nil {
|
||||
_ = watcher.Close()
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Watcher{watcher: watcher}, nil
|
||||
}
|
||||
|
||||
// Close stops watching the directory.
|
||||
func (w *Watcher) Close() error {
|
||||
return w.watcher.Close()
|
||||
}
|
||||
|
||||
// CallWhenQuiet calls changed each time the directory has gone quietTime
|
||||
// without a change, after a change to a path for which counts is true, until
|
||||
// ctx is done. It also calls failed with each error the watch reports. The
|
||||
// wait starts at once, as if for a change, so that a change made before the
|
||||
// watch began is taken in too.
|
||||
func (w *Watcher) CallWhenQuiet(
|
||||
ctx context.Context,
|
||||
counts func(path string) bool,
|
||||
changed func(),
|
||||
failed func(error),
|
||||
) {
|
||||
callWhenQuiet(
|
||||
ctx, w.watcher.Events, w.watcher.Errors, counts, changed, failed,
|
||||
)
|
||||
}
|
||||
|
||||
// callWhenQuiet is CallWhenQuiet, for the changes from events and the
|
||||
// errors from errs.
|
||||
func callWhenQuiet(
|
||||
ctx context.Context,
|
||||
events <-chan fsnotify.Event,
|
||||
errs <-chan error,
|
||||
counts func(path string) bool,
|
||||
changed func(),
|
||||
failed func(error),
|
||||
) {
|
||||
quiet := time.NewTimer(quietTime)
|
||||
defer quiet.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case event := <-events:
|
||||
if counts(event.Name) {
|
||||
quiet.Reset(quietTime)
|
||||
}
|
||||
case <-quiet.C:
|
||||
changed()
|
||||
case err := <-errs:
|
||||
failed(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,167 +0,0 @@
|
||||
package dirwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
)
|
||||
|
||||
// errWatch is an error the watch reports.
|
||||
var errWatch = errors.New("too many changes")
|
||||
|
||||
func TestDirectoryThatCannotBeWatchedIsAnError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := New(filepath.Join(t.TempDir(), "missing"))
|
||||
if err == nil {
|
||||
t.Error("a missing directory is watched, want an error")
|
||||
}
|
||||
}
|
||||
|
||||
// The tests below run callWhenQuiet in a synctest bubble, where time is a
|
||||
// clock of the test's own: time.Sleep moves it on at once, and
|
||||
// synctest.Wait returns once callWhenQuiet waits again, so that every call
|
||||
// due by then is made. The test sends the changes itself, as the watch of
|
||||
// a directory cannot run in a bubble.
|
||||
|
||||
func TestSecondChangeStartsTheWaitAgain(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
loop := start(t, everyPath)
|
||||
|
||||
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 0)
|
||||
|
||||
// The second part starts the wait again.
|
||||
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 0)
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCalledOnceQuietAfterTheStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
// An edit saved before the watch began sends no change, and is
|
||||
// taken in by this call.
|
||||
loop := start(t, everyPath)
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 1)
|
||||
|
||||
// Long after, it has been called only once.
|
||||
time.Sleep(time.Hour)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestChangeThatDoesNotCountIsNoChange(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
loop := start(t, func(path string) bool {
|
||||
return path == "ipinfo_lite.mmdb"
|
||||
})
|
||||
|
||||
// The wait that starts with the watch ends with a call.
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
|
||||
loop.changes <- fsnotify.Event{Name: "other.mmdb", Op: fsnotify.Create}
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 1)
|
||||
|
||||
loop.changes <- fsnotify.Event{Name: "ipinfo_lite.mmdb", Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 2)
|
||||
})
|
||||
}
|
||||
|
||||
func TestErrorsOfTheWatchHandedOn(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
loop := start(t, everyPath)
|
||||
|
||||
loop.errs <- errWatch
|
||||
|
||||
synctest.Wait()
|
||||
|
||||
if !errors.Is(loop.failure, errWatch) {
|
||||
t.Errorf("failed with %v, want %v", loop.failure, errWatch)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// loop is a callWhenQuiet that a test runs.
|
||||
type loop struct {
|
||||
// changes and errs send it changes and errors.
|
||||
changes chan fsnotify.Event
|
||||
errs chan error
|
||||
// calls counts its calls of changed, and failure is the last error it
|
||||
// handed to failed.
|
||||
calls atomic.Int32
|
||||
failure error
|
||||
}
|
||||
|
||||
// start runs callWhenQuiet, with counts, until the test ends.
|
||||
func start(t *testing.T, counts func(path string) bool) *loop {
|
||||
t.Helper()
|
||||
|
||||
l := &loop{changes: make(chan fsnotify.Event), errs: make(chan error)}
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
callWhenQuiet(ctx, l.changes, l.errs, counts,
|
||||
func() { l.calls.Add(1) },
|
||||
func(err error) { l.failure = err },
|
||||
)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
<-stopped
|
||||
})
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
// everyPath counts a change to any path.
|
||||
func everyPath(string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// wantCalls checks how many times changed has been called.
|
||||
func (l *loop) wantCalls(t *testing.T, want int32) {
|
||||
t.Helper()
|
||||
|
||||
got := l.calls.Load()
|
||||
if got != want {
|
||||
t.Errorf("changed called %d times, want %d", got, want)
|
||||
}
|
||||
}
|
||||
+49
-18
@@ -13,13 +13,18 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/oschwald/maxminddb-golang/v2"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||
)
|
||||
|
||||
// quietTime is how long the lookup database must go without a change
|
||||
// before it is read again, so that a file still being copied in is read
|
||||
// only once whole.
|
||||
const quietTime = 2 * time.Second
|
||||
|
||||
// FileParams are what OpenFile needs.
|
||||
type FileParams struct {
|
||||
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
||||
@@ -117,13 +122,20 @@ func (f *File) ReadFailures() int {
|
||||
}
|
||||
|
||||
// Watch watches the directory of the lookup database until ctx is done,
|
||||
// and reads the file again once it has gone without a change for a short
|
||||
// time, after it is replaced, written or removed, and after Watch starts
|
||||
// watching, as dirwatch describes. A change to another file in its
|
||||
// directory does not count. If the directory cannot be watched, that is
|
||||
// logged, and the database read at start stays in use.
|
||||
// and reads the file again once it has gone without a change for
|
||||
// quietTime, after it is replaced, written or removed, and after Watch
|
||||
// starts watching. If the directory cannot be watched, that is logged, and
|
||||
// the database read at start stays in use.
|
||||
func (f *File) Watch(ctx context.Context) {
|
||||
watcher, err := dirwatch.New(filepath.Dir(f.params.Path))
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err == nil {
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
err = watcher.Add(filepath.Dir(f.params.Path))
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
f.params.ProcessLog.Error(
|
||||
"cannot watch the lookup database for replacements",
|
||||
@@ -134,22 +146,41 @@ func (f *File) Watch(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
||||
"file", f.params.Path)
|
||||
|
||||
watcher.CallWhenQuiet(ctx, f.isDatabase, f.readAgain, func(err error) {
|
||||
f.params.ProcessLog.Warn("watching the lookup database failed",
|
||||
"error", err.Error())
|
||||
})
|
||||
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
||||
}
|
||||
|
||||
// isDatabase reports whether path names the lookup database.
|
||||
func (f *File) isDatabase(path string) bool {
|
||||
return filepath.Clean(path) == filepath.Clean(f.params.Path)
|
||||
// readAfterChanges reads the lookup database again once quietTime has
|
||||
// passed without a change to it from events, until ctx is done, and logs
|
||||
// the errors from errs. A change to another file in its directory does not
|
||||
// count. The wait starts at once, as if for a change, so that a file
|
||||
// replaced after OpenFile read it, and before its directory was watched,
|
||||
// is read too.
|
||||
func (f *File) readAfterChanges(
|
||||
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
||||
) {
|
||||
path := filepath.Clean(f.params.Path)
|
||||
|
||||
quiet := time.NewTimer(quietTime)
|
||||
defer quiet.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case event := <-events:
|
||||
if filepath.Clean(event.Name) == path {
|
||||
quiet.Reset(quietTime)
|
||||
}
|
||||
case <-quiet.C:
|
||||
f.readAgain()
|
||||
case err := <-errs:
|
||||
f.params.ProcessLog.Warn("watching the lookup database failed",
|
||||
"error", err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readAgain reads the lookup database again, in place of the one in use,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package lookup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/maxmind/mmdbwriter/mmdbtype"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
@@ -131,48 +133,66 @@ func TestFileThatCannotBeReadIsAnError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
||||
t.Parallel()
|
||||
// The tests below run readAfterChanges in a synctest bubble, where time is
|
||||
// a clock of the test's own: time.Sleep moves it on at once, and
|
||||
// synctest.Wait returns once readAfterChanges waits again, so that every
|
||||
// reading due by then is done. The test sends the changes itself, as the
|
||||
// watch of a directory cannot run in a bubble.
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||
writeDatabase(t, path, "DE")
|
||||
|
||||
f := openFile(t, path, newQueue())
|
||||
|
||||
for changed, want := range map[string]bool{
|
||||
path: true,
|
||||
dir + "/./ipinfo_lite.mmdb": true,
|
||||
filepath.Join(dir, "other.mmdb"): false,
|
||||
} {
|
||||
if f.isDatabase(changed) != want {
|
||||
t.Errorf(
|
||||
"a change to %s counts: %t, want %t",
|
||||
changed,
|
||||
!want,
|
||||
want,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The tests below run in a synctest bubble, where time is a clock of the
|
||||
// test's own, so that the time a file is read can be checked. The wait for
|
||||
// the directory to go quiet before a reading is tested in dirwatch.
|
||||
|
||||
func TestReplacementIsReadAgain(t *testing.T) {
|
||||
func TestReplacementCopiedOverTheFileInTwoPartsIsReadOnlyWhole(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||
writeDatabase(t, path, "DE")
|
||||
|
||||
queue := newQueue()
|
||||
f := openFile(t, path, queue)
|
||||
changes := watch(t, f)
|
||||
|
||||
writeDatabase(t, path, "KP")
|
||||
time.Sleep(time.Hour)
|
||||
f.readAgain()
|
||||
other := filepath.Join(dir, "replacement.mmdb")
|
||||
writeDatabase(t, other, "KP")
|
||||
|
||||
//nolint:gosec // a file the test wrote
|
||||
replacement, err := os.ReadFile(other)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", other, err)
|
||||
}
|
||||
|
||||
// The file in use is overwritten in place, and keeps giving what
|
||||
// it gave. Its first part alone is not a .mmdb file.
|
||||
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
||||
if err != nil {
|
||||
t.Fatalf("create %s: %v", path, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
half := len(replacement) / 2
|
||||
write(t, file, replacement[:half])
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "DE")
|
||||
|
||||
// The second part starts the wait again.
|
||||
write(t, file, replacement[half:])
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "DE")
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "KP")
|
||||
|
||||
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
||||
@@ -199,10 +219,15 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
||||
queue := newQueue()
|
||||
f := openFile(t, path, queue)
|
||||
read := f.LastRead()
|
||||
changes := watch(t, f)
|
||||
|
||||
writeFile(t, path, "not a lookup database\n")
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
// Long after, the replacement has been read once.
|
||||
time.Sleep(time.Hour)
|
||||
f.readAgain()
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "DE")
|
||||
|
||||
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
||||
@@ -215,7 +240,7 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
||||
}
|
||||
|
||||
wantAlerts(t, queue, alerts.Alert{
|
||||
Time: read.Add(time.Hour),
|
||||
Time: read.Add(quietTime),
|
||||
Event: alerts.EventFileError,
|
||||
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
||||
Detail: map[string]any{
|
||||
@@ -227,6 +252,55 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
||||
})
|
||||
}
|
||||
|
||||
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||
writeDatabase(t, path, "DE")
|
||||
|
||||
f := openFile(t, path, newQueue())
|
||||
changes := watch(t, f)
|
||||
|
||||
// The wait that starts with the watch ends with a reading.
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
writeDatabase(t, path, "KP")
|
||||
|
||||
changes <- fsnotify.Event{Name: filepath.Join(dir, "other.mmdb"), Op: fsnotify.Create}
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "DE")
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "KP")
|
||||
})
|
||||
}
|
||||
|
||||
func TestReplacementSavedBeforeTheWatchStartsIsRead(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||
writeDatabase(t, path, "DE")
|
||||
|
||||
f := openFile(t, path, newQueue())
|
||||
|
||||
// Saved after OpenFile read the file, and before its directory was
|
||||
// watched, so that no change is seen for it.
|
||||
writeDatabase(t, path, "KP")
|
||||
watch(t, f)
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "KP")
|
||||
})
|
||||
}
|
||||
|
||||
// newQueue returns a queue of alerts for a webhook that is never sent
|
||||
// them, so that they wait in it for the test to look at.
|
||||
func newQueue() *alerts.Queue {
|
||||
@@ -270,6 +344,28 @@ func openFile(t *testing.T, path string, queue *alerts.Queue) *File {
|
||||
return f
|
||||
}
|
||||
|
||||
// watch runs f's readAfterChanges until the test ends, and returns the
|
||||
// channel that sends it changes.
|
||||
func watch(t *testing.T, f *File) chan<- fsnotify.Event {
|
||||
t.Helper()
|
||||
|
||||
changes := make(chan fsnotify.Event)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
f.readAfterChanges(ctx, changes, nil)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
<-stopped
|
||||
})
|
||||
|
||||
return changes
|
||||
}
|
||||
|
||||
// wantCountry checks the country f gives testClient.
|
||||
func wantCountry(t *testing.T, f *File, want string) {
|
||||
t.Helper()
|
||||
@@ -305,3 +401,13 @@ func writeFile(t *testing.T, path, content string) {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// write writes data to the end of file.
|
||||
func write(t *testing.T, file *os.File, data []byte) {
|
||||
t.Helper()
|
||||
|
||||
_, err := file.Write(data)
|
||||
if err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
+268
-393
@@ -15,33 +15,12 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
)
|
||||
|
||||
// Params are what New builds the metrics from. Each part is nil while it is
|
||||
// off, and a part that is nil adds no metric of its own.
|
||||
type Params struct {
|
||||
// BusiestCount is how many countries and how many AS numbers get series
|
||||
// of their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS).
|
||||
BusiestCount int
|
||||
// InstanceName is the label instance every metric carries
|
||||
// (SWW_WAF_INSTANCE_NAME).
|
||||
InstanceName string
|
||||
// Now returns the time at which Bans counts the bans active.
|
||||
Now func() time.Time
|
||||
|
||||
Bans banLedger
|
||||
Clients sized
|
||||
Rules sized
|
||||
Lists fetchedLists
|
||||
DNSBL dnsblZones
|
||||
AbuseIPDB abuseIPDBChecker
|
||||
Alerts alertQueue
|
||||
LookupFile lookupDatabase
|
||||
RemoteLog remoteLogSender
|
||||
}
|
||||
|
||||
// Metrics are sww's metrics. They are safe for concurrent use.
|
||||
type Metrics struct {
|
||||
// registry gives every metric registered with it the label instance.
|
||||
@@ -57,10 +36,12 @@ type Metrics struct {
|
||||
rateLimitHits *prometheus.CounterVec
|
||||
offences *prometheus.CounterVec
|
||||
coreRuleSetMatches *prometheus.CounterVec
|
||||
ruleMatches *prometheus.CounterVec
|
||||
reputationHits *prometheus.CounterVec
|
||||
countries *busiest
|
||||
asns *busiest
|
||||
// ruleMatches are made by AddRules, and reputationHits by
|
||||
// AddReputation.
|
||||
ruleMatches *prometheus.CounterVec
|
||||
reputationHits *prometheus.CounterVec
|
||||
countries *busiest
|
||||
asns *busiest
|
||||
|
||||
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
|
||||
// that failed. RequestsWithoutGeoJSAnswer are the requests that needed
|
||||
@@ -78,18 +59,20 @@ type Metrics struct {
|
||||
stateFileEditsSetAside *prometheus.CounterVec
|
||||
}
|
||||
|
||||
// New returns the metrics, with the Go runtime's and the process's own, and
|
||||
// those read from each part in params as the metrics are asked for.
|
||||
// New returns the metrics, with the Go runtime's and the process's own.
|
||||
// busiestCount is how many countries and how many AS numbers get series of
|
||||
// their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS). Every metric
|
||||
// carries instanceName (SWW_WAF_INSTANCE_NAME) as its label instance.
|
||||
//
|
||||
//nolint:funlen // a few lines for each metric, a list that grows with them
|
||||
func New(params Params) *Metrics {
|
||||
func New(busiestCount int, instanceName string) *Metrics {
|
||||
byStatus := []string{"status_class", "action"}
|
||||
byFile := []string{"file"}
|
||||
registry := prometheus.NewRegistry()
|
||||
|
||||
m := &Metrics{
|
||||
registry: prometheus.WrapRegistererWith(
|
||||
prometheus.Labels{"instance": params.InstanceName}, registry),
|
||||
prometheus.Labels{"instance": instanceName}, registry),
|
||||
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
||||
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "sww_requests_in_flight",
|
||||
@@ -130,19 +113,8 @@ func New(params Params) *Metrics {
|
||||
"and the rule's id.",
|
||||
[]string{"mode", "rule_id"},
|
||||
),
|
||||
ruleMatches: counterVec(
|
||||
"sww_rule_matches_total",
|
||||
"Requests that matched a rule of the rule files, by its id and action.",
|
||||
[]string{"rule_id", "action"},
|
||||
),
|
||||
reputationHits: counterVec(
|
||||
"sww_reputation_hits_total",
|
||||
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
||||
"by the list's URL, the zone, or abuseipdb.",
|
||||
[]string{sourceLabel},
|
||||
),
|
||||
countries: newCountries(params.BusiestCount),
|
||||
asns: newASNs(params.BusiestCount),
|
||||
countries: newCountries(busiestCount),
|
||||
asns: newASNs(busiestCount),
|
||||
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sww_geojs_requests_total",
|
||||
Help: "Requests to GeoJS.",
|
||||
@@ -194,7 +166,6 @@ func New(params Params) *Metrics {
|
||||
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
||||
m.requestDuration, m.upstreamDuration,
|
||||
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
||||
m.ruleMatches, m.reputationHits,
|
||||
m.countries, m.asns,
|
||||
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
||||
m.stateFileWrites, m.stateFileWriteFailures,
|
||||
@@ -202,45 +173,261 @@ func New(params Params) *Metrics {
|
||||
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
||||
)
|
||||
|
||||
if params.Bans != nil {
|
||||
m.addBans(params.Bans, params.Now)
|
||||
}
|
||||
|
||||
if params.Clients != nil {
|
||||
m.addClients(params.Clients)
|
||||
}
|
||||
|
||||
if params.Rules != nil {
|
||||
m.addRules(params.Rules)
|
||||
}
|
||||
|
||||
if params.Lists != nil {
|
||||
m.addLists(params.Lists)
|
||||
}
|
||||
|
||||
if params.DNSBL != nil {
|
||||
m.addDNSBL(params.DNSBL)
|
||||
}
|
||||
|
||||
if params.AbuseIPDB != nil {
|
||||
m.addAbuseIPDB(params.AbuseIPDB)
|
||||
}
|
||||
|
||||
if params.Alerts != nil {
|
||||
m.addAlerts(params.Alerts)
|
||||
}
|
||||
|
||||
if params.LookupFile != nil {
|
||||
m.addLookupFile(params.LookupFile)
|
||||
}
|
||||
|
||||
if params.RemoteLog != nil {
|
||||
m.addRemoteLog(params.RemoteLog)
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
// AddBansAndClients adds the metrics read from the ledger and the table
|
||||
// of clients as the metrics are asked for: the bans made since the start,
|
||||
// by cause, the bans active and permanent at now, and the clients in the
|
||||
// table.
|
||||
func (m *Metrics) AddBansAndClients(
|
||||
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
||||
) {
|
||||
for _, cause := range []string{
|
||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
||||
} {
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_bans_made_total",
|
||||
Help: "Bans made, by cause.",
|
||||
ConstLabels: prometheus.Labels{"cause": cause},
|
||||
}, func() float64 {
|
||||
return float64(ledger.Made(cause))
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_active_bans",
|
||||
Help: "Bans active now, the permanent ones included.",
|
||||
}, func() float64 {
|
||||
active, _ := ledger.Count(now())
|
||||
|
||||
return float64(active)
|
||||
}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_permanent_bans",
|
||||
Help: "Permanent bans not lifted.",
|
||||
}, func() float64 {
|
||||
_, permanent := ledger.Count(now())
|
||||
|
||||
return float64(permanent)
|
||||
}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_tracked_clients",
|
||||
Help: "Clients in the table of clients.",
|
||||
}, func() float64 {
|
||||
return float64(limiter.Len())
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// AddRules adds the metrics of the rule files: the requests that matched
|
||||
// each rule, which RuleMatched counts, and the rules loaded from
|
||||
// ruleFiles, read as the metrics are asked for. It is called once, before
|
||||
// RuleMatched.
|
||||
func (m *Metrics) AddRules(ruleFiles *rules.Files) {
|
||||
m.ruleMatches = counterVec(
|
||||
"sww_rule_matches_total",
|
||||
"Requests that matched a rule of the rule files, by its id and action.",
|
||||
[]string{"rule_id", "action"},
|
||||
)
|
||||
|
||||
m.registry.MustRegister(m.ruleMatches,
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_rules_loaded",
|
||||
Help: "Rules loaded from the rule files.",
|
||||
}, func() float64 {
|
||||
return float64(ruleFiles.Len())
|
||||
}))
|
||||
}
|
||||
|
||||
// AddRemoteLog adds the metrics of sending the log lines to
|
||||
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
||||
// lines sent, those dropped, and those waiting in the buffer.
|
||||
func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_remote_log_lines_sent_total",
|
||||
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
||||
}, func() float64 {
|
||||
return float64(remote.Sent())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_remote_log_lines_dropped_total",
|
||||
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
||||
"whose sending failed.",
|
||||
}, func() float64 {
|
||||
return float64(remote.Dropped())
|
||||
}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_remote_log_lines_waiting",
|
||||
Help: "Log lines in the buffer, waiting to be sent.",
|
||||
}, func() float64 {
|
||||
return float64(remote.Depth())
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// AddLookupFile adds the metrics of the lookup database, read as the
|
||||
// metrics are asked for: when the file in use was read, which lastRead
|
||||
// returns, and the replacements of it that could not be read, which
|
||||
// readFailures returns.
|
||||
func (m *Metrics) AddLookupFile(
|
||||
lastRead func() time.Time,
|
||||
readFailures func() int,
|
||||
) {
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
||||
Help: "When the lookup database in use was read, in seconds since 1970.",
|
||||
}, func() float64 {
|
||||
return float64(lastRead().Unix())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_lookup_database_read_failures_total",
|
||||
Help: "Replacements of the lookup database that could not be read.",
|
||||
}, func() float64 {
|
||||
return float64(readFailures())
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
||||
// DNSBL zone, its key masked, or abuseipdb.
|
||||
const sourceLabel = "source"
|
||||
|
||||
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
||||
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
||||
// the fetches that failed and when the copy in use was fetched, and for a
|
||||
// zone, the queries made and those that failed. It is called once, before
|
||||
// ReputationHit.
|
||||
func (m *Metrics) AddReputation(
|
||||
lists *reputation.Lists,
|
||||
dnsbl *reputation.DNSBL,
|
||||
) {
|
||||
m.reputationHits = counterVec(
|
||||
"sww_reputation_hits_total",
|
||||
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
||||
"by the list's URL, the zone, or abuseipdb.",
|
||||
[]string{sourceLabel},
|
||||
)
|
||||
m.registry.MustRegister(m.reputationHits)
|
||||
|
||||
for _, zone := range dnsbl.Zones() {
|
||||
source := prometheus.Labels{
|
||||
sourceLabel: reputation.MaskZoneKey(zone),
|
||||
}
|
||||
|
||||
m.addReputationQueries(
|
||||
source,
|
||||
func() int { return dnsbl.Queries(zone) },
|
||||
)
|
||||
m.addReputationFailures(
|
||||
source,
|
||||
func() int { return dnsbl.Failures(zone) },
|
||||
)
|
||||
}
|
||||
|
||||
for _, listURL := range lists.URLs() {
|
||||
source := prometheus.Labels{sourceLabel: listURL}
|
||||
|
||||
m.addReputationFailures(
|
||||
source,
|
||||
func() int { return lists.Failures(listURL) },
|
||||
)
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
||||
Help: "When the copy of the list in use was fetched, in seconds since " +
|
||||
"1970, or 0 while there is none.",
|
||||
ConstLabels: source,
|
||||
}, func() float64 {
|
||||
fetched := lists.Fetched(listURL)
|
||||
if fetched.IsZero() {
|
||||
return 0
|
||||
}
|
||||
|
||||
return float64(fetched.Unix())
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// AddAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
||||
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
||||
// that failed, and how many checks the day's budget has left. It is
|
||||
// called once, after AddReputation, while SWW_WAF_ABUSEIPDB_KEY is set.
|
||||
func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) {
|
||||
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
||||
|
||||
m.addReputationQueries(source, abuseIPDB.Checked)
|
||||
m.addReputationFailures(source, abuseIPDB.Failures)
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_reputation_daily_budget_remaining",
|
||||
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
||||
ConstLabels: source,
|
||||
}, func() float64 {
|
||||
return float64(abuseIPDB.BudgetLeft())
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// ReputationHit counts a request whose client source lists: a blocklist,
|
||||
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
||||
func (m *Metrics) ReputationHit(source string) {
|
||||
m.reputationHits.WithLabelValues(source).Inc()
|
||||
}
|
||||
|
||||
// AddAlerts adds the metrics of the alerts sent to each destination set,
|
||||
// read from queue as the metrics are asked for, by destination: the
|
||||
// alerts sent, the requests to the destination that failed, the alerts
|
||||
// held back, which are the same for every destination, and those
|
||||
// dropped. With no destination set, it adds none.
|
||||
func (m *Metrics) AddAlerts(queue *alerts.Queue) {
|
||||
for _, name := range queue.DestinationsSet() {
|
||||
destination := prometheus.Labels{"destination": name}
|
||||
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_alerts_sent_total",
|
||||
Help: "Alerts the destination took.",
|
||||
ConstLabels: destination,
|
||||
}, func() float64 {
|
||||
return float64(queue.Counts(name).Sent)
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_alert_send_failures_total",
|
||||
Help: "Requests to the destination that failed.",
|
||||
ConstLabels: destination,
|
||||
}, func() float64 {
|
||||
return float64(queue.Counts(name).SendFailures)
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_alerts_suppressed_total",
|
||||
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
||||
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
||||
ConstLabels: destination,
|
||||
}, func() float64 {
|
||||
return float64(queue.Suppressed())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_alerts_dropped_total",
|
||||
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
||||
"alerts given up as the destination refused them.",
|
||||
ConstLabels: destination,
|
||||
}, func() float64 {
|
||||
return float64(queue.Counts(name).Dropped)
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// ServeHTTP answers with the metrics in the Prometheus text format.
|
||||
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
m.handler.ServeHTTP(w, r)
|
||||
@@ -310,12 +497,6 @@ func (m *Metrics) CoreRuleSetMatched(mode string, id int) {
|
||||
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
||||
}
|
||||
|
||||
// ReputationHit counts a request whose client source lists: a blocklist,
|
||||
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
||||
func (m *Metrics) ReputationHit(source string) {
|
||||
m.reputationHits.WithLabelValues(source).Inc()
|
||||
}
|
||||
|
||||
// StateFileWritten counts a write of the state or cache file name, of size
|
||||
// bytes, that ended with err.
|
||||
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
||||
@@ -347,312 +528,6 @@ func (m *Metrics) StateFileEditSetAside(name string) {
|
||||
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
||||
}
|
||||
|
||||
// banLedger is what the metrics read from the ledger of bans.
|
||||
type banLedger interface {
|
||||
// Made returns the bans made since the start with cause.
|
||||
Made(cause string) int
|
||||
// Count returns the bans active at now, the permanent ones included,
|
||||
// and the permanent ones.
|
||||
Count(now time.Time) (active, permanent int)
|
||||
}
|
||||
|
||||
// addBans adds the metrics read from ledger as the metrics are asked for:
|
||||
// the bans made since the start, by cause, and the bans active and
|
||||
// permanent at now.
|
||||
func (m *Metrics) addBans(ledger banLedger, now func() time.Time) {
|
||||
for _, cause := range []string{
|
||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
||||
} {
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_bans_made_total",
|
||||
Help: "Bans made, by cause.",
|
||||
ConstLabels: prometheus.Labels{"cause": cause},
|
||||
}, func() float64 {
|
||||
return float64(ledger.Made(cause))
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_active_bans",
|
||||
Help: "Bans active now, the permanent ones included.",
|
||||
}, func() float64 {
|
||||
active, _ := ledger.Count(now())
|
||||
|
||||
return float64(active)
|
||||
}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_permanent_bans",
|
||||
Help: "Permanent bans not lifted.",
|
||||
}, func() float64 {
|
||||
_, permanent := ledger.Count(now())
|
||||
|
||||
return float64(permanent)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// sized is what the metrics read from the table of clients and from the
|
||||
// rule files.
|
||||
type sized interface {
|
||||
// Len returns how many clients the table holds, or how many rules were
|
||||
// loaded from the rule files.
|
||||
Len() int
|
||||
}
|
||||
|
||||
// addClients adds the metric of the clients in table, read as the metrics
|
||||
// are asked for.
|
||||
func (m *Metrics) addClients(table sized) {
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_tracked_clients",
|
||||
Help: "Clients in the table of clients.",
|
||||
}, func() float64 {
|
||||
return float64(table.Len())
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// addRules adds the metric of the rules loaded from files, read as the
|
||||
// metrics are asked for.
|
||||
func (m *Metrics) addRules(files sized) {
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_rules_loaded",
|
||||
Help: "Rules loaded from the rule files.",
|
||||
}, func() float64 {
|
||||
return float64(files.Len())
|
||||
}))
|
||||
}
|
||||
|
||||
// remoteLogSender is what the metrics read from what sends the log lines to
|
||||
// SWW_WAF_LOG_REMOTE_URL.
|
||||
type remoteLogSender interface {
|
||||
// Sent returns the lines sent.
|
||||
Sent() int64
|
||||
// Dropped returns the lines dropped.
|
||||
Dropped() int64
|
||||
// Depth returns the lines waiting in the buffer.
|
||||
Depth() int
|
||||
}
|
||||
|
||||
// addRemoteLog adds the metrics of sending the log lines to
|
||||
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
||||
// lines sent, those dropped, and those waiting in the buffer.
|
||||
func (m *Metrics) addRemoteLog(remote remoteLogSender) {
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_remote_log_lines_sent_total",
|
||||
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
||||
}, func() float64 {
|
||||
return float64(remote.Sent())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_remote_log_lines_dropped_total",
|
||||
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
||||
"whose sending failed.",
|
||||
}, func() float64 {
|
||||
return float64(remote.Dropped())
|
||||
}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_remote_log_lines_waiting",
|
||||
Help: "Log lines in the buffer, waiting to be sent.",
|
||||
}, func() float64 {
|
||||
return float64(remote.Depth())
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// lookupDatabase is what the metrics read from the lookup database.
|
||||
type lookupDatabase interface {
|
||||
// LastRead returns when the file in use was read.
|
||||
LastRead() time.Time
|
||||
// ReadFailures returns the replacements of the file that could not be
|
||||
// read.
|
||||
ReadFailures() int
|
||||
}
|
||||
|
||||
// addLookupFile adds the metrics of the lookup database, read from file as
|
||||
// the metrics are asked for: when the file in use was read, and the
|
||||
// replacements of it that could not be read.
|
||||
func (m *Metrics) addLookupFile(file lookupDatabase) {
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
||||
Help: "When the lookup database in use was read, in seconds since 1970.",
|
||||
}, func() float64 {
|
||||
return float64(file.LastRead().Unix())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_lookup_database_read_failures_total",
|
||||
Help: "Replacements of the lookup database that could not be read.",
|
||||
}, func() float64 {
|
||||
return float64(file.ReadFailures())
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
||||
// DNSBL zone, its key masked, or abuseipdb.
|
||||
const sourceLabel = "source"
|
||||
|
||||
// dnsblZones is what the metrics read from what asks the DNSBL zones.
|
||||
type dnsblZones interface {
|
||||
// Zones returns the zones asked, their keys unmasked.
|
||||
Zones() []string
|
||||
// Queries returns the queries made to zone.
|
||||
Queries(zone string) int
|
||||
// Failures returns the queries to zone that failed.
|
||||
Failures(zone string) int
|
||||
}
|
||||
|
||||
// addDNSBL adds the metrics of each zone dnsbl asks, by source, the zone,
|
||||
// its key masked as reputation.MaskZoneKey masks it, read as the metrics are
|
||||
// asked for: the queries made and those that failed.
|
||||
func (m *Metrics) addDNSBL(dnsbl dnsblZones) {
|
||||
for _, zone := range dnsbl.Zones() {
|
||||
source := prometheus.Labels{
|
||||
sourceLabel: reputation.MaskZoneKey(zone),
|
||||
}
|
||||
|
||||
m.addReputationQueries(
|
||||
source,
|
||||
func() int { return dnsbl.Queries(zone) },
|
||||
)
|
||||
m.addReputationFailures(
|
||||
source,
|
||||
func() int { return dnsbl.Failures(zone) },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// fetchedLists is what the metrics read from the lists fetched from URLs.
|
||||
type fetchedLists interface {
|
||||
// URLs returns the lists' URLs.
|
||||
URLs() []string
|
||||
// Failures returns the fetches of the list at listURL that failed.
|
||||
Failures(listURL string) int
|
||||
// Fetched returns when the copy in use of the list at listURL was
|
||||
// fetched, or the zero time while there is none.
|
||||
Fetched(listURL string) time.Time
|
||||
}
|
||||
|
||||
// addLists adds the metrics of each list fetched from a URL, by source, the
|
||||
// list's URL, read from lists as the metrics are asked for: the fetches
|
||||
// that failed and when the copy in use was fetched.
|
||||
func (m *Metrics) addLists(lists fetchedLists) {
|
||||
for _, listURL := range lists.URLs() {
|
||||
source := prometheus.Labels{sourceLabel: listURL}
|
||||
|
||||
m.addReputationFailures(
|
||||
source,
|
||||
func() int { return lists.Failures(listURL) },
|
||||
)
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
||||
Help: "When the copy of the list in use was fetched, in seconds since " +
|
||||
"1970, or 0 while there is none.",
|
||||
ConstLabels: source,
|
||||
}, func() float64 {
|
||||
fetched := lists.Fetched(listURL)
|
||||
if fetched.IsZero() {
|
||||
return 0
|
||||
}
|
||||
|
||||
return float64(fetched.Unix())
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// abuseIPDBChecker is what the metrics read from what checks clients with
|
||||
// AbuseIPDB.
|
||||
type abuseIPDBChecker interface {
|
||||
// Checked returns the checks made.
|
||||
Checked() int
|
||||
// Failures returns the checks that failed.
|
||||
Failures() int
|
||||
// BudgetLeft returns how many checks the day's budget has left.
|
||||
BudgetLeft() int
|
||||
}
|
||||
|
||||
// addAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
||||
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
||||
// that failed, and how many checks the day's budget has left.
|
||||
func (m *Metrics) addAbuseIPDB(abuseIPDB abuseIPDBChecker) {
|
||||
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
||||
|
||||
m.addReputationQueries(source, abuseIPDB.Checked)
|
||||
m.addReputationFailures(source, abuseIPDB.Failures)
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "sww_reputation_daily_budget_remaining",
|
||||
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
||||
ConstLabels: source,
|
||||
}, func() float64 {
|
||||
return float64(abuseIPDB.BudgetLeft())
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// alertQueue is what the metrics read from the queue of alerts.
|
||||
type alertQueue interface {
|
||||
// DestinationsSet returns the names of the destinations set.
|
||||
DestinationsSet() []string
|
||||
// Counts returns the alerts sent to the destination name, the requests
|
||||
// to it that failed, and the alerts dropped.
|
||||
Counts(name string) alerts.Counts
|
||||
// Suppressed returns the alerts held back.
|
||||
Suppressed() int64
|
||||
}
|
||||
|
||||
// addAlerts adds the metrics of the alerts sent to each destination set,
|
||||
// read from queue as the metrics are asked for, by destination: the
|
||||
// alerts sent, the requests to the destination that failed, the alerts
|
||||
// held back, which are the same for every destination, and those
|
||||
// dropped. With no destination set, it adds none.
|
||||
func (m *Metrics) addAlerts(queue alertQueue) {
|
||||
for _, name := range queue.DestinationsSet() {
|
||||
destination := prometheus.Labels{"destination": name}
|
||||
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_alerts_sent_total",
|
||||
Help: "Alerts the destination took.",
|
||||
ConstLabels: destination,
|
||||
}, func() float64 {
|
||||
return float64(queue.Counts(name).Sent)
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_alert_send_failures_total",
|
||||
Help: "Requests to the destination that failed.",
|
||||
ConstLabels: destination,
|
||||
}, func() float64 {
|
||||
return float64(queue.Counts(name).SendFailures)
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_alerts_suppressed_total",
|
||||
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
||||
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
||||
ConstLabels: destination,
|
||||
}, func() float64 {
|
||||
return float64(queue.Suppressed())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "sww_alerts_dropped_total",
|
||||
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
||||
"alerts given up as the destination refused them.",
|
||||
ConstLabels: destination,
|
||||
}, func() float64 {
|
||||
return float64(queue.Counts(name).Dropped)
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// count counts a request that has ended, as RequestEnded and RequestReported
|
||||
// describe.
|
||||
func (m *Metrics) count(
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
package metrics_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
)
|
||||
|
||||
const (
|
||||
listURL = "https://lists.example/block.txt"
|
||||
zone = "dnsbl.example"
|
||||
destination = "webhook"
|
||||
)
|
||||
|
||||
// Each stand-in answers with a number of its own, so that each metric
|
||||
// shows which part it was read from.
|
||||
|
||||
type ledgerStandIn struct{}
|
||||
|
||||
func (ledgerStandIn) Made(string) int { return 1 }
|
||||
|
||||
func (ledgerStandIn) Count(time.Time) (int, int) { return 2, 1 }
|
||||
|
||||
type clientTableStandIn struct{}
|
||||
|
||||
func (clientTableStandIn) Len() int { return 3 }
|
||||
|
||||
type ruleFilesStandIn struct{}
|
||||
|
||||
func (ruleFilesStandIn) Len() int { return 4 }
|
||||
|
||||
type listsStandIn struct{}
|
||||
|
||||
func (listsStandIn) URLs() []string { return []string{listURL} }
|
||||
|
||||
func (listsStandIn) Failures(string) int { return 5 }
|
||||
|
||||
func (listsStandIn) Fetched(string) time.Time { return time.Time{} }
|
||||
|
||||
type dnsblStandIn struct{}
|
||||
|
||||
func (dnsblStandIn) Zones() []string { return []string{zone} }
|
||||
|
||||
func (dnsblStandIn) Queries(string) int { return 6 }
|
||||
|
||||
func (dnsblStandIn) Failures(string) int { return 0 }
|
||||
|
||||
type abuseIPDBStandIn struct{}
|
||||
|
||||
func (abuseIPDBStandIn) Checked() int { return 0 }
|
||||
|
||||
func (abuseIPDBStandIn) Failures() int { return 0 }
|
||||
|
||||
func (abuseIPDBStandIn) BudgetLeft() int { return 7 }
|
||||
|
||||
type alertQueueStandIn struct{}
|
||||
|
||||
func (alertQueueStandIn) DestinationsSet() []string {
|
||||
return []string{destination}
|
||||
}
|
||||
|
||||
func (alertQueueStandIn) Counts(string) alerts.Counts {
|
||||
return alerts.Counts{Sent: 8}
|
||||
}
|
||||
|
||||
func (alertQueueStandIn) Suppressed() int64 { return 0 }
|
||||
|
||||
type lookupDatabaseStandIn struct{}
|
||||
|
||||
func (lookupDatabaseStandIn) LastRead() time.Time { return time.Time{} }
|
||||
|
||||
func (lookupDatabaseStandIn) ReadFailures() int { return 9 }
|
||||
|
||||
type remoteLogStandIn struct{}
|
||||
|
||||
func (remoteLogStandIn) Sent() int64 { return 10 }
|
||||
|
||||
func (remoteLogStandIn) Dropped() int64 { return 0 }
|
||||
|
||||
func (remoteLogStandIn) Depth() int { return 0 }
|
||||
|
||||
func TestMetricsAreReadFromEachPart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m := metrics.New(metrics.Params{
|
||||
BusiestCount: 1,
|
||||
InstanceName: "app",
|
||||
Now: func() time.Time { return time.Time{} },
|
||||
Bans: ledgerStandIn{},
|
||||
Clients: clientTableStandIn{},
|
||||
Rules: ruleFilesStandIn{},
|
||||
Lists: listsStandIn{},
|
||||
DNSBL: dnsblStandIn{},
|
||||
AbuseIPDB: abuseIPDBStandIn{},
|
||||
Alerts: alertQueueStandIn{},
|
||||
LookupFile: lookupDatabaseStandIn{},
|
||||
RemoteLog: remoteLogStandIn{},
|
||||
})
|
||||
scraped := scrape(t, m)
|
||||
|
||||
for _, line := range []string{
|
||||
`sww_bans_made_total{cause="limit",instance="app"} 1`,
|
||||
`sww_active_bans{instance="app"} 2`,
|
||||
`sww_tracked_clients{instance="app"} 3`,
|
||||
`sww_rules_loaded{instance="app"} 4`,
|
||||
`sww_reputation_failures_total{instance="app",source="` + listURL + `"} 5`,
|
||||
`sww_reputation_queries_total{instance="app",source="` + zone + `"} 6`,
|
||||
`sww_reputation_daily_budget_remaining{instance="app",source="abuseipdb"} 7`,
|
||||
`sww_alerts_sent_total{destination="` + destination + `",instance="app"} 8`,
|
||||
`sww_lookup_database_read_failures_total{instance="app"} 9`,
|
||||
`sww_remote_log_lines_sent_total{instance="app"} 10`,
|
||||
} {
|
||||
if !strings.Contains(scraped, "\n"+line+"\n") {
|
||||
t.Errorf("metrics\n%s\nwant\n%s", scraped, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartThatIsOffAddsNoMetric(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
scraped := scrape(t, metrics.New(metrics.Params{
|
||||
BusiestCount: 1,
|
||||
InstanceName: "app",
|
||||
}))
|
||||
|
||||
for _, name := range []string{
|
||||
"sww_bans_made_total",
|
||||
"sww_tracked_clients",
|
||||
"sww_rules_loaded",
|
||||
"sww_reputation_",
|
||||
"sww_alert",
|
||||
"sww_lookup_database_",
|
||||
"sww_remote_log_",
|
||||
} {
|
||||
if strings.Contains(scraped, name) {
|
||||
t.Errorf("metrics\n%s\nshow %s", scraped, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// scrape returns the metrics m serves.
|
||||
func scrape(t *testing.T, m *metrics.Metrics) string {
|
||||
t.Helper()
|
||||
|
||||
scraped := httptest.NewRecorder()
|
||||
m.ServeHTTP(
|
||||
scraped,
|
||||
httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, "/", http.NoBody,
|
||||
),
|
||||
)
|
||||
|
||||
return scraped.Body.String()
|
||||
}
|
||||
@@ -445,9 +445,12 @@ func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||
scores: map[string]int64{suspect: 100},
|
||||
}
|
||||
checker := newAbuseIPDB(abuseIPDB, p)
|
||||
m := metrics.New(metrics.Params{
|
||||
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
||||
})
|
||||
m := metrics.New(1, "app")
|
||||
m.AddReputation(
|
||||
reputation.New(params()),
|
||||
reputation.NewDNSBL(dnsblParams()),
|
||||
)
|
||||
m.AddAbuseIPDB(checker)
|
||||
|
||||
// One check that AbuseIPDB answers, and one it refuses with an answer
|
||||
// that names the key.
|
||||
@@ -500,9 +503,12 @@ func TestMetricsCountTheChecksTheFailuresAndTheBudgetLeft(t *testing.T) {
|
||||
p := abuseIPDBParams()
|
||||
p.DailyBudget = 5
|
||||
checker := newAbuseIPDB(abuseIPDB, p)
|
||||
m := metrics.New(metrics.Params{
|
||||
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
||||
})
|
||||
m := metrics.New(1, "app")
|
||||
m.AddReputation(
|
||||
reputation.New(params()),
|
||||
reputation.NewDNSBL(dnsblParams()),
|
||||
)
|
||||
m.AddAbuseIPDB(checker)
|
||||
|
||||
// One check that AbuseIPDB answers, and one that fails.
|
||||
hitFrom(t, checker, suspect, true)
|
||||
@@ -805,9 +811,6 @@ func wantBudgetLeft(t *testing.T, checker *reputation.AbuseIPDB, want int) {
|
||||
}
|
||||
}
|
||||
|
||||
// instanceName is the label instance of the metrics the tests build.
|
||||
const instanceName = "app"
|
||||
|
||||
// scrapeMetrics returns the metrics m serves.
|
||||
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
|
||||
t.Helper()
|
||||
|
||||
@@ -358,9 +358,8 @@ func TestMetricsCountEachZonesQueriesAndThoseThatFailed(t *testing.T) {
|
||||
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
||||
}}
|
||||
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
||||
m := metrics.New(metrics.Params{
|
||||
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
||||
})
|
||||
m := metrics.New(1, "app")
|
||||
m.AddReputation(reputation.New(params()), dnsbl)
|
||||
|
||||
wantZones(t, dnsbl, listed)
|
||||
synctest.Wait()
|
||||
@@ -405,9 +404,8 @@ func TestZoneKeyIsMaskedInTheVerdictsTheFailuresAndTheMetrics(t *testing.T) {
|
||||
"99.2.0.192." + keyed + ".": {addrs: []string{listing}},
|
||||
"100.2.0.192." + keyed + ".": {rcode: serverFailure},
|
||||
}}, p)
|
||||
m := metrics.New(metrics.Params{
|
||||
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
||||
})
|
||||
m := metrics.New(1, "app")
|
||||
m.AddReputation(reputation.New(params()), dnsbl)
|
||||
|
||||
// Both clients are asked about before either answer comes, so that
|
||||
// the failure does not keep the zone from the other query.
|
||||
|
||||
@@ -2,7 +2,6 @@ package rproxy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
@@ -23,9 +22,7 @@ type requestBody struct {
|
||||
// body is the visitor's body, ending in an *http.MaxBytesError past
|
||||
// SWW_RPROXY_MAX_REQUEST_SIZE.
|
||||
body io.ReadCloser
|
||||
// refuse refuses the request with a status, as the request's own refuse
|
||||
// does.
|
||||
refuse func(status int)
|
||||
rq *request
|
||||
// held is the part read before the request goes to the app, which Read gives
|
||||
// first.
|
||||
held []byte
|
||||
@@ -71,7 +68,7 @@ func (b *requestBody) Read(p []byte) (int, error) {
|
||||
case errors.Is(err, io.EOF):
|
||||
b.received.Store(true)
|
||||
case errors.As(err, &tooLarge):
|
||||
b.refuse(http.StatusRequestEntityTooLarge)
|
||||
b.rq.refuse(http.StatusRequestEntityTooLarge)
|
||||
}
|
||||
|
||||
return n, err
|
||||
@@ -89,13 +86,7 @@ type responseBody struct {
|
||||
// body is the app's body, ending in an *http.MaxBytesError past
|
||||
// SWW_RPROXY_MAX_RESPONSE_SIZE.
|
||||
body io.ReadCloser
|
||||
// refuse refuses the request with a status, as the request's own refuse
|
||||
// does.
|
||||
refuse func(status int)
|
||||
// responseReceived is called once the app's whole answer has arrived.
|
||||
responseReceived func()
|
||||
// ctx is the visitor's request's context.
|
||||
ctx context.Context //nolint:containedctx // the body ends with the request
|
||||
rq *request
|
||||
}
|
||||
|
||||
// Read reads from the app's body.
|
||||
@@ -109,15 +100,15 @@ func (b *responseBody) Read(p []byte) (int, error) {
|
||||
|
||||
switch {
|
||||
case errors.Is(err, io.EOF):
|
||||
b.responseReceived()
|
||||
b.rq.responseReceived()
|
||||
case errors.As(err, &tooLarge):
|
||||
b.refuse(http.StatusBadGateway)
|
||||
b.rq.refuse(http.StatusBadGateway)
|
||||
|
||||
return n, errResponseTooLarge
|
||||
case b.ctx.Err() == nil:
|
||||
case b.rq.in.Context().Err() == nil:
|
||||
// The answer broke off, not because the visitor went away. If a time
|
||||
// limit cut it, that refusal came first and is the one kept.
|
||||
b.refuse(http.StatusBadGateway)
|
||||
b.rq.refuse(http.StatusBadGateway)
|
||||
}
|
||||
|
||||
return n, err
|
||||
|
||||
@@ -122,8 +122,8 @@ func (h *handler) newRequest(
|
||||
|
||||
if r.Body != http.NoBody {
|
||||
rq.body = &requestBody{
|
||||
body: limitBody(r.Body, h.config.MaxRequestSize),
|
||||
refuse: rq.refuse,
|
||||
body: limitBody(r.Body, h.config.MaxRequestSize),
|
||||
rq: rq,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -291,12 +291,7 @@ func (rq *request) modifyResponse(res *http.Response) error {
|
||||
// the request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts.
|
||||
rq.startResponseTimer(requestEnd)
|
||||
|
||||
res.Body = &responseBody{
|
||||
body: limitBody(res.Body, maxBytes),
|
||||
refuse: rq.refuse,
|
||||
responseReceived: rq.responseReceived,
|
||||
ctx: rq.in.Context(),
|
||||
}
|
||||
res.Body = &responseBody{body: limitBody(res.Body, maxBytes), rq: rq}
|
||||
rq.out.afterStatus = rq.setResponseDeadline
|
||||
|
||||
return nil
|
||||
|
||||
+48
-19
@@ -18,11 +18,12 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"golang.org/x/net/http/httpguts"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
||||
)
|
||||
|
||||
// The actions a rule takes when it matches.
|
||||
@@ -39,6 +40,12 @@ const (
|
||||
// extension ends the name of every rule file.
|
||||
const extension = ".rules"
|
||||
|
||||
// quietTime is how long SWW_WAF_RULE_FILES_DIR must go without a change before
|
||||
// the rule files are read again, so that a file still being written, such
|
||||
// as one saved in place, appended to or copied in with scp, is read only
|
||||
// once whole.
|
||||
const quietTime = 2 * time.Second
|
||||
|
||||
// headerTarget starts the target that is one request header,
|
||||
// header:<Name>.
|
||||
const headerTarget = "header:"
|
||||
@@ -169,18 +176,26 @@ func (f *Files) Len() int {
|
||||
}
|
||||
|
||||
// Watch watches Dir until ctx is done, and reads the rule files again
|
||||
// once Dir has had no change for a short time, after one is edited, added
|
||||
// or removed, and after Watch starts watching, as dirwatch describes. If
|
||||
// they then hold an error, the rules stay as they were, the error is
|
||||
// logged with its file and line, and the files are read again after the
|
||||
// next change. If Dir cannot be watched, that is logged, and the rules
|
||||
// stay as they were loaded. While Enabled is false, Watch returns at once.
|
||||
// once Dir has had no change for quietTime, after one is edited, added or
|
||||
// removed, and after Watch starts watching. If they then hold an error,
|
||||
// the rules stay as they were, the error is logged with its file and
|
||||
// line, and the files are read again after the next change. If Dir cannot
|
||||
// be watched, that is logged, and the rules stay as they were loaded.
|
||||
// While Enabled is false, Watch returns at once.
|
||||
func (f *Files) Watch(ctx context.Context) {
|
||||
if !f.params.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
watcher, err := dirwatch.New(f.params.Dir)
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err == nil {
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
err = watcher.Add(f.params.Dir)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
||||
"error", err.Error())
|
||||
@@ -188,22 +203,36 @@ func (f *Files) Watch(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
f.params.ProcessLog.Info("watching the rule files for edits",
|
||||
"directory", f.params.Dir)
|
||||
|
||||
// A change to any file in Dir counts.
|
||||
watcher.CallWhenQuiet(ctx,
|
||||
func(string) bool { return true },
|
||||
f.readAgain,
|
||||
func(err error) {
|
||||
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
||||
}
|
||||
|
||||
// readAfterChanges reads the rule files again once quietTime has passed
|
||||
// without a change from events, until ctx is done, and logs the errors
|
||||
// from errs. The wait starts at once, as if for a change, so that an edit
|
||||
// saved after Load read the files, and before Dir was watched, is taken
|
||||
// in too.
|
||||
func (f *Files) readAfterChanges(
|
||||
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
||||
) {
|
||||
quiet := time.NewTimer(quietTime)
|
||||
defer quiet.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-events:
|
||||
quiet.Reset(quietTime)
|
||||
case <-quiet.C:
|
||||
f.readAgain()
|
||||
case err := <-errs:
|
||||
f.params.ProcessLog.Warn("watching the rule files failed",
|
||||
"error", err.Error())
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readAgain reads the rule files again, in place of the rules loaded, or
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package rules
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
)
|
||||
|
||||
// The tests below run readAfterChanges in a synctest bubble, where time is
|
||||
// a clock of the test's own: time.Sleep moves it on at once, and
|
||||
// synctest.Wait returns once readAfterChanges waits again, so that every
|
||||
// reading due by then is done. The test sends the changes itself, as the
|
||||
// watch of a directory cannot run in a bubble.
|
||||
|
||||
func TestFileWrittenInTwoPartsTakenInOnlyWhole(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "50-app.rules")
|
||||
writeFile(t, path, "first path block ^/first\n")
|
||||
files := load(t, dir)
|
||||
changes := run(t, files)
|
||||
|
||||
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
// The first part ends in the middle of a ban rule's regex, which,
|
||||
// read then, would ban every request.
|
||||
write(t, file, "first path block ^/first\nprobe path ban ^/")
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantMatched(t, files, "/anything")
|
||||
|
||||
// The second part starts the wait again.
|
||||
write(t, file, `\.env$`+"\n")
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantMatched(t, files, "/.env")
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantMatched(t, files, "/.env", "probe")
|
||||
wantMatched(t, files, "/anything")
|
||||
})
|
||||
}
|
||||
|
||||
func TestEditSavedBeforeTheWatchStartsTakenIn(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "50-app.rules")
|
||||
writeFile(t, path, "first path block ^/first\n")
|
||||
files := load(t, dir)
|
||||
|
||||
// Saved after Load read the files, and before the directory was
|
||||
// watched, so that no change is seen for it.
|
||||
writeFile(t, path, "first path block ^/edited\n")
|
||||
run(t, files)
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
wantMatched(t, files, "/edited", "first")
|
||||
})
|
||||
}
|
||||
|
||||
// load loads the rules in dir.
|
||||
func load(t *testing.T, dir string) *Files {
|
||||
t.Helper()
|
||||
|
||||
files, err := Load(Params{
|
||||
Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler),
|
||||
Alerts: alerts.New(alerts.Params{}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// run runs files' readAfterChanges until the test ends, and returns the
|
||||
// channel that sends it changes.
|
||||
func run(t *testing.T, files *Files) chan<- fsnotify.Event {
|
||||
t.Helper()
|
||||
|
||||
changes := make(chan fsnotify.Event)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
files.readAfterChanges(ctx, changes, nil)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
<-stopped
|
||||
})
|
||||
|
||||
return changes
|
||||
}
|
||||
|
||||
// writeFile writes content to the file at path.
|
||||
func writeFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
|
||||
err := os.WriteFile(path, []byte(content), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// write writes text to the end of file.
|
||||
func write(t *testing.T, file *os.File, text string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := file.WriteString(text)
|
||||
if err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// wantMatched checks the ids of the rules that a GET request for path
|
||||
// matches, in order.
|
||||
func wantMatched(t *testing.T, files *Files, path string, want ...string) {
|
||||
t.Helper()
|
||||
|
||||
r := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
"http://app.example"+path, nil)
|
||||
|
||||
matched := files.Match(r)
|
||||
|
||||
got := make([]string, 0, len(matched))
|
||||
for _, rule := range matched {
|
||||
got = append(got, rule.ID)
|
||||
}
|
||||
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("%s matched %v, want %v", path, got, want)
|
||||
}
|
||||
}
|
||||
@@ -2430,7 +2430,7 @@ func midnight() time.Time {
|
||||
// counts, with thresholds fill does not reach.
|
||||
func newParams(dir string) state.Params {
|
||||
discard := slog.New(slog.DiscardHandler)
|
||||
m := metrics.New(metrics.Params{BusiestCount: 1, InstanceName: "app"})
|
||||
m := metrics.New(1, "app")
|
||||
queue := alerts.New(alerts.Params{
|
||||
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
||||
Events: alerts.Events(),
|
||||
|
||||
+30
-52
@@ -10,14 +10,13 @@ package sww
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
@@ -27,7 +26,14 @@ import (
|
||||
// know.
|
||||
const usageStatus = 2
|
||||
|
||||
var errNoCommand = errors.New("no command given")
|
||||
// usage is what sww prints then.
|
||||
const usage = `usage: sww <command>
|
||||
|
||||
commands:
|
||||
rproxy the reverse proxy in front of one app
|
||||
waf decides for rproxy, and keeps bans, counters and history
|
||||
healthcheck the container's health check
|
||||
`
|
||||
|
||||
// Params are what RunWAF and RunRProxy need from the process.
|
||||
type Params struct {
|
||||
@@ -45,9 +51,14 @@ type Params struct {
|
||||
|
||||
// Main runs the command os.Args names, and returns the process's exit status.
|
||||
// `sww rproxy` and `sww waf` run until SIGTERM or SIGINT.
|
||||
// With no command, or one it does not know, cobra prints the error and the
|
||||
// commands to stderr, and Main returns usageStatus.
|
||||
// With no command, or one it does not know, it prints the commands and returns
|
||||
// usageStatus.
|
||||
func Main(version string) int {
|
||||
args := os.Args[1:]
|
||||
if len(args) == 0 {
|
||||
return printUsage(os.Stderr)
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(),
|
||||
syscall.SIGTERM, os.Interrupt)
|
||||
defer stop()
|
||||
@@ -58,54 +69,21 @@ func Main(version string) int {
|
||||
Stdout: os.Stdout,
|
||||
}
|
||||
|
||||
// status is what the command that ran returns.
|
||||
status := 0
|
||||
|
||||
root := &cobra.Command{
|
||||
Use: "sww",
|
||||
Short: "a web application firewall",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(*cobra.Command, []string) error {
|
||||
return errNoCommand
|
||||
},
|
||||
switch args[0] {
|
||||
case "rproxy":
|
||||
return RunRProxy(ctx, params)
|
||||
case "waf":
|
||||
return RunWAF(ctx, params)
|
||||
case "healthcheck":
|
||||
return HealthCheck(ctx, args[1:], params.Viper, os.Stderr)
|
||||
default:
|
||||
return printUsage(os.Stderr)
|
||||
}
|
||||
root.CompletionOptions.DisableDefaultCmd = true
|
||||
}
|
||||
|
||||
root.AddCommand(
|
||||
&cobra.Command{
|
||||
Use: "rproxy",
|
||||
Short: "the reverse proxy in front of one app",
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
status = RunRProxy(cmd.Context(), params)
|
||||
},
|
||||
},
|
||||
&cobra.Command{
|
||||
Use: "waf",
|
||||
Short: "decides for rproxy, and keeps bans, counters and history",
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
status = RunWAF(cmd.Context(), params)
|
||||
},
|
||||
},
|
||||
&cobra.Command{
|
||||
Use: "healthcheck",
|
||||
Short: "the container's health check",
|
||||
// HealthCheck refuses every argument itself, flags included.
|
||||
DisableFlagParsing: true,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
status = HealthCheck(
|
||||
cmd.Context(),
|
||||
args,
|
||||
params.Viper,
|
||||
cmd.ErrOrStderr(),
|
||||
)
|
||||
},
|
||||
},
|
||||
)
|
||||
// printUsage writes the commands to w, and returns usageStatus.
|
||||
func printUsage(w io.Writer) int {
|
||||
_, _ = fmt.Fprint(w, usage)
|
||||
|
||||
err := root.ExecuteContext(ctx)
|
||||
if err != nil {
|
||||
return usageStatus
|
||||
}
|
||||
|
||||
return status
|
||||
return usageStatus
|
||||
}
|
||||
+11
-18
@@ -392,33 +392,26 @@ func newMetrics(
|
||||
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
||||
remote *remotelog.Sender,
|
||||
) *metrics.Metrics {
|
||||
params := metrics.Params{
|
||||
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
||||
InstanceName: cfg.InstanceName,
|
||||
Now: utcNow,
|
||||
Bans: ledger,
|
||||
Clients: limiter,
|
||||
Rules: ruleFiles,
|
||||
Lists: lists,
|
||||
DNSBL: dnsbl,
|
||||
Alerts: alertQueue,
|
||||
m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName)
|
||||
m.AddBansAndClients(ledger, limiter, utcNow)
|
||||
m.AddRules(ruleFiles)
|
||||
m.AddReputation(lists, dnsbl)
|
||||
|
||||
if cfg.AbuseIPDBKey != "" {
|
||||
m.AddAbuseIPDB(abuseIPDB)
|
||||
}
|
||||
|
||||
// A nil pointer in an interface field is not a nil interface, so a part
|
||||
// that is off is left out rather than set to nil.
|
||||
if cfg.AbuseIPDBKey != "" {
|
||||
params.AbuseIPDB = abuseIPDB
|
||||
}
|
||||
m.AddAlerts(alertQueue)
|
||||
|
||||
if lookupFile != nil {
|
||||
params.LookupFile = lookupFile
|
||||
m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
|
||||
}
|
||||
|
||||
if remote != nil {
|
||||
params.RemoteLog = remote
|
||||
m.AddRemoteLog(remote)
|
||||
}
|
||||
|
||||
return metrics.New(params)
|
||||
return m
|
||||
}
|
||||
|
||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||
|
||||
+29
-88
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -165,104 +164,46 @@ func wantUTC(t *testing.T, where string, value any) {
|
||||
func TestMainWithoutACommandItKnowsPrintsTheCommands(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// completion is cobra's own command, which sww turns off.
|
||||
for _, command := range []string{"", "serve", "completion"} {
|
||||
t.Run("command "+command, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
code, stdout, stderr := runMain(t, command)
|
||||
|
||||
if code != 2 {
|
||||
t.Errorf("exit status %d, want 2", code)
|
||||
}
|
||||
|
||||
wantTheCommands(t, "stderr", stderr)
|
||||
|
||||
if stdout != "" {
|
||||
t.Errorf("stdout %q, want nothing", stdout)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMainHelpPrintsTheCommands(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, command := range []string{"help", "--help"} {
|
||||
t.Run("command "+command, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
code, stdout, _ := runMain(t, command)
|
||||
|
||||
if code != 0 {
|
||||
t.Errorf("exit status %d, want 0", code)
|
||||
}
|
||||
|
||||
wantTheCommands(t, "stdout", stdout)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// wantTheCommands checks that text, what Main wrote to where, names each
|
||||
// command.
|
||||
func wantTheCommands(t *testing.T, where, text string) {
|
||||
t.Helper()
|
||||
|
||||
for _, name := range []string{"rproxy", wafCommand, "healthcheck"} {
|
||||
if !strings.Contains(text, name) {
|
||||
t.Errorf("%s %q does not name %s", where, text, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMainHealthCheckRefusesAnArgument(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, argument := range []string{"now", "--x", "-h"} {
|
||||
t.Run("argument "+argument, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
code, stdout, stderr := runMain(t, "healthcheck "+argument)
|
||||
|
||||
want := "sww healthcheck: unexpected argument \"" +
|
||||
argument + "\"\n"
|
||||
if code != 1 || stderr != want || stdout != "" {
|
||||
t.Errorf("exit status %d, stdout %q and stderr %q, "+
|
||||
"want 1, nothing and %q", code, stdout, stderr, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// runMain runs Main with command in a process of its own, and returns its
|
||||
// exit status and what it wrote to stdout and stderr.
|
||||
func runMain(t *testing.T, command string) (int, string, string) {
|
||||
t.Helper()
|
||||
|
||||
binary, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("find the test binary: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), waitLimit)
|
||||
defer cancel()
|
||||
for _, command := range []string{"", "serve"} {
|
||||
t.Run("command "+command, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
ctx, cancel := context.WithTimeout(t.Context(), waitLimit)
|
||||
defer cancel()
|
||||
|
||||
//nolint:gosec // this test binary
|
||||
cmd := exec.CommandContext(ctx, binary)
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
cmd.Env = append(os.Environ(), commandForMain+"="+command)
|
||||
cmd := exec.CommandContext(ctx, binary)
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
|
||||
err = cmd.Run()
|
||||
cmd.Env = append(os.Environ(), commandForMain+"="+command)
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if err != nil && !errors.As(err, &exitErr) {
|
||||
t.Fatalf("run: %v", err)
|
||||
err := cmd.Run()
|
||||
|
||||
if code := cmd.ProcessState.ExitCode(); code != 2 {
|
||||
t.Errorf("exit status %d (%v), want 2", code, err)
|
||||
}
|
||||
|
||||
for _, name := range []string{"rproxy", wafCommand, "healthcheck"} {
|
||||
if !strings.Contains(stderr.String(), name) {
|
||||
t.Errorf(
|
||||
"stderr %q does not name %s",
|
||||
stderr.String(),
|
||||
name,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if stdout.Len() != 0 {
|
||||
t.Errorf("stdout %q, want nothing", stdout.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return cmd.ProcessState.ExitCode(), stdout.String(), stderr.String()
|
||||
}
|
||||
|
||||
// ask sends waf at addr rproxy's question about a request for path from
|
||||
|
||||
+8
-12
@@ -45,11 +45,8 @@ func (a activeBan) check(
|
||||
}
|
||||
|
||||
ban, banned, madePermanent := check(rq.client, now)
|
||||
|
||||
var expires string
|
||||
if banned {
|
||||
expires = banExpires(ban)
|
||||
rq.line.BanExpires = expires
|
||||
rq.line.BanExpires = banExpires(ban)
|
||||
}
|
||||
|
||||
if madePermanent {
|
||||
@@ -64,7 +61,7 @@ func (a activeBan) check(
|
||||
return &verdict{
|
||||
action: banAction(ban.Permanent()),
|
||||
figures: map[string]any{
|
||||
figureNetblock: ban.Netblock.String(), "ban_expires": expires,
|
||||
figureNetblock: ban.Netblock.String(), "ban_expires": rq.line.BanExpires,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -151,7 +148,6 @@ func (b banMaker) banForLimit(
|
||||
rq.line.LimitHit = hit.Window
|
||||
}
|
||||
|
||||
rq.brokeLimit = true
|
||||
rq.line.Offence = requestlog.OffenceLimit
|
||||
|
||||
b.metrics.LimitHit(hit)
|
||||
@@ -263,9 +259,9 @@ func (b banMaker) withClientNotes(
|
||||
status int,
|
||||
netblock netip.Prefix,
|
||||
) bans.Notes {
|
||||
notes.ASN = rq.lookupAnswer.ASN
|
||||
notes.ASName = rq.lookupAnswer.ASName
|
||||
notes.Country = rq.lookupAnswer.Country
|
||||
notes.ASN = rq.line.ASN
|
||||
notes.ASName = rq.line.ASName
|
||||
notes.Country = rq.line.Country
|
||||
notes.Reputation = rq.reputation
|
||||
notes.Request = rq.noted(now, status)
|
||||
notes.Requests = b.netblockRequests(netblock)
|
||||
@@ -332,11 +328,11 @@ func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
|
||||
func (rq *request) noted(now time.Time, status int) bans.Request {
|
||||
return bans.Request{
|
||||
Time: now,
|
||||
Method: rq.method,
|
||||
Host: rq.site,
|
||||
Method: rq.line.Method,
|
||||
Host: rq.line.Host,
|
||||
Path: rq.requestURI,
|
||||
Status: status,
|
||||
UserAgent: rq.userAgent,
|
||||
UserAgent: rq.line.UserAgent,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@ type biasedThresholds struct {
|
||||
func (b biasedThresholds) limitPercentages(
|
||||
rq *request,
|
||||
) (percentage, percentage) {
|
||||
asn, country := rq.lookupAnswer.ASN, rq.lookupAnswer.Country
|
||||
asn, country := rq.line.ASN, rq.line.Country
|
||||
|
||||
unknown := percentage{percent: whole}
|
||||
if country == "" && !rq.skippedLookupWait {
|
||||
@@ -115,7 +115,7 @@ func (b biasedThresholds) limitPercentages(
|
||||
|
||||
otherTags := percentage{percent: whole}
|
||||
|
||||
for _, tag := range rq.tags {
|
||||
for _, tag := range rq.line.Tags {
|
||||
if tag != asnTag(asn) && tag != countryTag(country) {
|
||||
otherTags = lowest(
|
||||
otherTags,
|
||||
|
||||
@@ -33,7 +33,7 @@ func (b byteLimits) check(
|
||||
rq *request,
|
||||
now time.Time,
|
||||
) *verdict {
|
||||
if !rq.counted || rq.wouldAction != "" {
|
||||
if !rq.counted || rq.line.WouldAction != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -121,9 +121,9 @@ func (c coreRuleSet) alertCoreRuleSetMatch(
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
|
||||
ASN: rq.lookupAnswer.ASN,
|
||||
ASName: rq.lookupAnswer.ASName,
|
||||
Country: rq.lookupAnswer.Country,
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
Reason: "scored by the Core Rule Set at or over " +
|
||||
"SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD",
|
||||
Detail: detail,
|
||||
|
||||
@@ -27,7 +27,7 @@ func (c countries) check(
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
allowed := c.allowed
|
||||
country := rq.lookupAnswer.Country
|
||||
country := rq.line.Country
|
||||
|
||||
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
||||
return nil
|
||||
|
||||
@@ -94,11 +94,11 @@ func (rq *request) noteLookup(result lookupResult) {
|
||||
rq.line.ASName = rq.lookupAnswer.ASName
|
||||
rq.line.Country = rq.lookupAnswer.Country
|
||||
|
||||
if tag := asnTag(rq.lookupAnswer.ASN); tag != "" {
|
||||
if tag := asnTag(rq.line.ASN); tag != "" {
|
||||
rq.addTag(tag)
|
||||
}
|
||||
|
||||
if tag := countryTag(rq.lookupAnswer.Country); tag != "" {
|
||||
if tag := countryTag(rq.line.Country); tag != "" {
|
||||
rq.addTag(tag)
|
||||
}
|
||||
}
|
||||
|
||||
+20
-27
@@ -548,11 +548,16 @@ func addCoreRuleSet(t *testing.T, params *waf.Params) {
|
||||
params.CoreRuleSet = coreRuleSet
|
||||
}
|
||||
|
||||
// addCounters adds to params the table of clients, the ledger and the
|
||||
// anomaly counters, as sww builds them from params' settings.
|
||||
// addCounters adds to params the metrics, the table of clients, the ledger
|
||||
// and the anomaly counters, as sww builds them from params' settings. The
|
||||
// metrics read the ledger, the table of clients and the rules.
|
||||
func addCounters(params *waf.Params) {
|
||||
cfg := params.Config
|
||||
|
||||
params.Metrics = metrics.New(
|
||||
cfg.MetricsBusiestCountriesAndASNs,
|
||||
cfg.InstanceName,
|
||||
)
|
||||
params.Limiter = ratelimit.New(ratelimit.Params{
|
||||
Limits: ratelimit.Limits{
|
||||
PerMinute: cfg.RateLimitPerMinute,
|
||||
@@ -582,18 +587,26 @@ func addCounters(params *waf.Params) {
|
||||
NamedNetblocks: cfg.AnomalyWatchedNets,
|
||||
Alerts: params.Alerts,
|
||||
})
|
||||
|
||||
params.Metrics.AddBansAndClients(
|
||||
params.Ledger,
|
||||
params.Limiter,
|
||||
params.Now,
|
||||
)
|
||||
params.Metrics.AddRules(params.Rules)
|
||||
}
|
||||
|
||||
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
||||
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
||||
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
||||
// setting is set, the metrics newMetrics returns, and GeoJS at geojsURL, as
|
||||
// sww builds them from params' settings and the parts addCounters added.
|
||||
// setting is set, and GeoJS at geojsURL, as sww builds
|
||||
// them from params' settings and the parts addCounters added. The metrics
|
||||
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
|
||||
func addOutsideSources(
|
||||
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
||||
) {
|
||||
cfg := params.Config
|
||||
limiter, ledger := params.Limiter, params.Ledger
|
||||
m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger
|
||||
processLog, alertQueue := params.ProcessLog, params.Alerts
|
||||
|
||||
params.Lists = reputation.New(reputation.Params{
|
||||
@@ -642,9 +655,6 @@ func addOutsideSources(
|
||||
})
|
||||
}
|
||||
|
||||
m := newMetrics(params)
|
||||
params.Metrics = m
|
||||
|
||||
params.GeoJS = geojs.New(geojs.Params{
|
||||
URL: geojsURL,
|
||||
Answered: func(answer geojs.Answer) {
|
||||
@@ -657,29 +667,12 @@ func addOutsideSources(
|
||||
},
|
||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||
})
|
||||
}
|
||||
|
||||
// newMetrics returns the metrics, as sww builds them from params' settings,
|
||||
// read from the ledger, the table of clients, the rules, the lists, the
|
||||
// zones' verdicts, and AbuseIPDB's while its key is set.
|
||||
func newMetrics(params *waf.Params) *metrics.Metrics {
|
||||
cfg := params.Config
|
||||
metricsParams := metrics.Params{
|
||||
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
||||
InstanceName: cfg.InstanceName,
|
||||
Now: params.Now,
|
||||
Bans: params.Ledger,
|
||||
Clients: params.Limiter,
|
||||
Rules: params.Rules,
|
||||
Lists: params.Lists,
|
||||
DNSBL: params.DNSBL,
|
||||
}
|
||||
m.AddReputation(params.Lists, params.DNSBL)
|
||||
|
||||
if cfg.AbuseIPDBKey != "" {
|
||||
metricsParams.AbuseIPDB = params.AbuseIPDB
|
||||
m.AddAbuseIPDB(params.AbuseIPDB)
|
||||
}
|
||||
|
||||
return metrics.New(metricsParams)
|
||||
}
|
||||
|
||||
// newClient returns an HTTP client that sends requests as they are made,
|
||||
|
||||
@@ -54,7 +54,7 @@ func (r refusalLimit) check(
|
||||
// What the client was sent, or in observe mode would have been.
|
||||
status := rq.out.status
|
||||
|
||||
switch rq.wouldAction {
|
||||
switch rq.line.WouldAction {
|
||||
case requestlog.ActionBlockedByBlockRule,
|
||||
requestlog.ActionBlockedByCoreRuleSet:
|
||||
status = http.StatusForbidden
|
||||
|
||||
@@ -47,9 +47,9 @@ func (n reputationHitNoter) noteHit(
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
|
||||
ASN: rq.lookupAnswer.ASN,
|
||||
ASName: rq.lookupAnswer.ASName,
|
||||
Country: rq.lookupAnswer.Country,
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
Reason: reason,
|
||||
Detail: detail,
|
||||
})
|
||||
|
||||
+20
-42
@@ -24,64 +24,42 @@ type refusal struct {
|
||||
action string
|
||||
}
|
||||
|
||||
// decision is what the checks note of a request: its log line, what the
|
||||
// heuristics tell each other, and what counting the request needs once it has
|
||||
// ended. Nothing reads the log line back to decide. waf keeps a copy of it
|
||||
// until the request's report comes. It holds no credentials, since the log
|
||||
// line never does, and neither the request's objects nor their bodies.
|
||||
// decision is what the checks note of a request: its log line, and what
|
||||
// counting the request needs once it has ended. waf keeps a copy of it
|
||||
// until the request's report comes. It holds no credentials, since the log line
|
||||
// never does, and neither the request's objects nor their bodies.
|
||||
type decision struct {
|
||||
line requestlog.Line
|
||||
client netip.Addr
|
||||
// method and userAgent are the request's method and User-Agent, and
|
||||
// requestURI its path with the query, as a ban's notes keep them.
|
||||
// startRequest sets them.
|
||||
method, userAgent, requestURI string
|
||||
// tags are the request's tags, sorted, each once. addTag adds them, for
|
||||
// tagLists, routeWatchTags, noteLookup, dnsbl, abuseIPDB, ruleFiles and
|
||||
// coreRuleSet.
|
||||
tags []string
|
||||
// requestURI is the path with the query, as a ban's notes keep it.
|
||||
requestURI string
|
||||
// site is, for a question, the host the visitor asked for, which the
|
||||
// alerts and a ban's notes name, since one waf serves many sites.
|
||||
// answerQuestion sets it.
|
||||
// alerts name, since one waf serves many sites.
|
||||
site string
|
||||
// allowed is true for a question answered with DispositionAllow.
|
||||
allowed bool
|
||||
// lookedUp is true once the client's AS number and country have been
|
||||
// looked up, whether or not an answer was there, and lookupAnswer is
|
||||
// what the lookup gave then, the zero Answer while GeoJS had given none.
|
||||
// noteLookup sets them.
|
||||
lookedUp bool
|
||||
lookupAnswer geojs.Answer
|
||||
// skippedLookupWait is true for a request that did not wait for GeoJS and
|
||||
// found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it,
|
||||
// so that not waiting never lowers its client's limits. noteLookup sets
|
||||
// it.
|
||||
// so that not waiting never lowers its client's limits.
|
||||
skippedLookupWait bool
|
||||
// counted is true for a request the rate limits counted, whose bytes
|
||||
// the byte limits count once it has ended. rateLimitPercent and
|
||||
// bytesLimitPercent are then its client's limit percentages for the rate
|
||||
// limits and for the byte limits. rateLimits sets them.
|
||||
// limits and for the byte limits.
|
||||
counted bool
|
||||
rateLimitPercent, bytesLimitPercent percentage
|
||||
// attack is true for a request that matched a ban rule or asked for a
|
||||
// trap path, blockedByBlockRule for one a block rule refused, and
|
||||
// blockedByCoreRuleSet for one the Core Rule Set refused, each an offence
|
||||
// its client's history counts. trapPaths and ruleFiles set attack,
|
||||
// ruleFiles blockedByBlockRule, and coreRuleSet blockedByCoreRuleSet.
|
||||
// its client's history counts.
|
||||
attack, blockedByBlockRule, blockedByCoreRuleSet bool
|
||||
// wouldAction is, in observe mode, the action enforce mode would have
|
||||
// refused the request with, one of requestlog's Action constants, and ""
|
||||
// otherwise. check sets it.
|
||||
wouldAction string
|
||||
// brokeLimit is true for a request that broke a limit, an offence its
|
||||
// client's history counts. banForLimit sets it.
|
||||
brokeLimit bool
|
||||
// blocklisted is true once blocklists finds a blocklist that lists the
|
||||
// client, dnsblListed once dnsbl finds a DNSBL zone's verdict that does,
|
||||
// and abuseIPDBHit once abuseIPDB finds AbuseIPDB's score of it a hit.
|
||||
blocklisted, dnsblListed, abuseIPDBHit bool
|
||||
// reputation is the reputation sources that list the client, for the
|
||||
// notes of a ban the request makes. noteHit adds them.
|
||||
// notes of a ban the request makes.
|
||||
reputation []bans.ReputationHit
|
||||
}
|
||||
|
||||
@@ -94,7 +72,11 @@ type request struct {
|
||||
in *http.Request
|
||||
out *responseWriter
|
||||
|
||||
start time.Time
|
||||
// blocklisted is true once a blocklist is found to list the client,
|
||||
// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
|
||||
// AbuseIPDB's score of it is a hit.
|
||||
blocklisted, dnsblListed, abuseIPDBHit bool
|
||||
start time.Time
|
||||
// checked is when the checks were done.
|
||||
checked time.Time
|
||||
// refused is what the request was refused with, nil if it was not.
|
||||
@@ -124,8 +106,6 @@ func (h *handler) startRequest(
|
||||
rq := &request{
|
||||
decision: decision{
|
||||
client: client,
|
||||
method: r.Method,
|
||||
userAgent: r.UserAgent(),
|
||||
requestURI: r.URL.RequestURI(),
|
||||
line: requestlog.Line{
|
||||
Time: requestlog.FormatTime(start),
|
||||
@@ -171,12 +151,11 @@ func (h *handler) startRequest(
|
||||
}
|
||||
|
||||
// addTag adds tag to the request's tags, which stay sorted and hold each tag
|
||||
// once, and to its log line.
|
||||
// once.
|
||||
func (rq *request) addTag(tag string) {
|
||||
at, found := slices.BinarySearch(rq.tags, tag)
|
||||
at, found := slices.BinarySearch(rq.line.Tags, tag)
|
||||
if !found {
|
||||
rq.tags = slices.Insert(rq.tags, at, tag)
|
||||
rq.line.Tags = rq.tags
|
||||
rq.line.Tags = slices.Insert(rq.line.Tags, at, tag)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,7 +196,6 @@ func (rq *request) check(ctx context.Context) *refusal {
|
||||
|
||||
switch {
|
||||
case rq.h.config.Observe:
|
||||
rq.wouldAction = action
|
||||
rq.line.WouldAction = action
|
||||
|
||||
return nil
|
||||
@@ -393,7 +371,7 @@ func (rq *request) addToHistory() {
|
||||
Status: rq.out.status,
|
||||
RequestBytes: rq.requestBytes,
|
||||
ResponseBytes: rq.out.bytes,
|
||||
BrokeLimit: rq.brokeLimit,
|
||||
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
|
||||
Attack: rq.attack,
|
||||
BlockedByBlockRule: rq.blockedByBlockRule,
|
||||
BlockedByCoreRuleSet: rq.blockedByCoreRuleSet,
|
||||
|
||||
@@ -23,7 +23,7 @@ func (t tagDeny) check(
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
for _, tag := range rq.tags {
|
||||
for _, tag := range rq.line.Tags {
|
||||
if slices.Contains(t.tags, tag) {
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientDenied,
|
||||
|
||||
+2
-2
@@ -202,8 +202,8 @@ func (rq *request) allow() {
|
||||
|
||||
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
|
||||
if rq.h.config.AddLookupHeaders {
|
||||
answer.ASN = rq.lookupAnswer.ASN
|
||||
answer.Country = rq.lookupAnswer.Country
|
||||
answer.ASN = rq.line.ASN
|
||||
answer.Country = rq.line.Country
|
||||
}
|
||||
|
||||
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
|
||||
|
||||
Reference in new issue
Block a user