Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a46770ed58 |
No files matched your search
@@ -906,13 +906,9 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
- `internal/dirwatch`: watches a directory, so that `internal/rules` and
|
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
||||||
`internal/lookup` read their files again when they change.
|
`internal/metrics`: the state and cache files, the log lines, the syslog
|
||||||
- `internal/state`: the state and cache files.
|
sender and the metrics.
|
||||||
- `internal/requestlog`: the request log.
|
|
||||||
- `internal/processlog`: the process's own messages.
|
|
||||||
- `internal/remotelog`: the syslog sender.
|
|
||||||
- `internal/metrics`: the metrics.
|
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
|
|||||||
@@ -93,9 +93,6 @@ never as a time.
|
|||||||
imports no other package of this repository: `waf` gives it what it needs
|
imports no other package of this repository: `waf` gives it what it needs
|
||||||
to count and report failures.
|
to count and report failures.
|
||||||
- `internal/lookup`: looks clients up in the lookup database.
|
- `internal/lookup`: looks clients up in the lookup database.
|
||||||
- `internal/dirwatch`: watches a directory, and calls a function once it has
|
|
||||||
gone a short time without a change. `internal/rules` and `internal/lookup`
|
|
||||||
read their files again this way.
|
|
||||||
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
||||||
AbuseIPDB in the background, and keeps their answers. It also fetches
|
AbuseIPDB in the background, and keeps their answers. It also fetches
|
||||||
AbuseIPDB's blacklist, which `internal/waf` bans.
|
AbuseIPDB's blacklist, which `internal/waf` bans.
|
||||||
@@ -107,10 +104,8 @@ never as a time.
|
|||||||
its own.
|
its own.
|
||||||
- `internal/state`: reads the state and cache files at start, takes in edits
|
- `internal/state`: reads the state and cache files at start, takes in edits
|
||||||
while running, and writes the files when due and at the stop.
|
while running, and writes the files when due and at the stop.
|
||||||
- `internal/requestlog`: the request log, as JSON lines on stdout.
|
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
|
||||||
- `internal/processlog`: the process lines on stdout, which hold the
|
sending them to the syslog server.
|
||||||
process's own messages.
|
|
||||||
- `internal/remotelog`: sends the lines on stdout to the syslog server.
|
|
||||||
- `internal/metrics`: the metrics, in the Prometheus text format.
|
- `internal/metrics`: the metrics, in the Prometheus text format.
|
||||||
- `Dockerfile`: the lint and test phases, then the image.
|
- `Dockerfile`: the lint and test phases, then the image.
|
||||||
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
// Package dirwatch watches a directory, and calls a function once the
|
|
||||||
// directory has gone a short time without a change after one. The rule
|
|
||||||
// files and the lookup database are read again this way.
|
|
||||||
package dirwatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
)
|
|
||||||
|
|
||||||
// quietTime is how long the directory must go without a change before the
|
|
||||||
// function is called, so that a file still being written, such as one
|
|
||||||
// saved in place, appended to or copied in with scp, is read only once
|
|
||||||
// whole.
|
|
||||||
const quietTime = 2 * time.Second
|
|
||||||
|
|
||||||
// Watcher watches one directory. The directory is watched, rather than a
|
|
||||||
// file in it, so that a file replaced by a rename is seen.
|
|
||||||
type Watcher struct {
|
|
||||||
watcher *fsnotify.Watcher
|
|
||||||
}
|
|
||||||
|
|
||||||
// New starts watching dir, or returns why it cannot.
|
|
||||||
func New(dir string) (*Watcher, error) {
|
|
||||||
watcher, err := fsnotify.NewWatcher()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = watcher.Add(dir)
|
|
||||||
if err != nil {
|
|
||||||
_ = watcher.Close()
|
|
||||||
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Watcher{watcher: watcher}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close stops watching the directory.
|
|
||||||
func (w *Watcher) Close() error {
|
|
||||||
return w.watcher.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// CallWhenQuiet calls changed each time the directory has gone quietTime
|
|
||||||
// without a change, after a change to a path for which counts is true, until
|
|
||||||
// ctx is done. It also calls failed with each error the watch reports. The
|
|
||||||
// wait starts at once, as if for a change, so that a change made before the
|
|
||||||
// watch began is taken in too.
|
|
||||||
func (w *Watcher) CallWhenQuiet(
|
|
||||||
ctx context.Context,
|
|
||||||
counts func(path string) bool,
|
|
||||||
changed func(),
|
|
||||||
failed func(error),
|
|
||||||
) {
|
|
||||||
callWhenQuiet(
|
|
||||||
ctx, w.watcher.Events, w.watcher.Errors, counts, changed, failed,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// callWhenQuiet is CallWhenQuiet, for the changes from events and the
|
|
||||||
// errors from errs.
|
|
||||||
func callWhenQuiet(
|
|
||||||
ctx context.Context,
|
|
||||||
events <-chan fsnotify.Event,
|
|
||||||
errs <-chan error,
|
|
||||||
counts func(path string) bool,
|
|
||||||
changed func(),
|
|
||||||
failed func(error),
|
|
||||||
) {
|
|
||||||
quiet := time.NewTimer(quietTime)
|
|
||||||
defer quiet.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case event := <-events:
|
|
||||||
if counts(event.Name) {
|
|
||||||
quiet.Reset(quietTime)
|
|
||||||
}
|
|
||||||
case <-quiet.C:
|
|
||||||
changed()
|
|
||||||
case err := <-errs:
|
|
||||||
failed(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,167 +0,0 @@
|
|||||||
package dirwatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"path/filepath"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errWatch is an error the watch reports.
|
|
||||||
var errWatch = errors.New("too many changes")
|
|
||||||
|
|
||||||
func TestDirectoryThatCannotBeWatchedIsAnError(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := New(filepath.Join(t.TempDir(), "missing"))
|
|
||||||
if err == nil {
|
|
||||||
t.Error("a missing directory is watched, want an error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The tests below run callWhenQuiet in a synctest bubble, where time is a
|
|
||||||
// clock of the test's own: time.Sleep moves it on at once, and
|
|
||||||
// synctest.Wait returns once callWhenQuiet waits again, so that every call
|
|
||||||
// due by then is made. The test sends the changes itself, as the watch of
|
|
||||||
// a directory cannot run in a bubble.
|
|
||||||
|
|
||||||
func TestSecondChangeStartsTheWaitAgain(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
loop := start(t, everyPath)
|
|
||||||
|
|
||||||
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 0)
|
|
||||||
|
|
||||||
// The second part starts the wait again.
|
|
||||||
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 0)
|
|
||||||
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 1)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCalledOnceQuietAfterTheStart(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
// An edit saved before the watch began sends no change, and is
|
|
||||||
// taken in by this call.
|
|
||||||
loop := start(t, everyPath)
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 1)
|
|
||||||
|
|
||||||
// Long after, it has been called only once.
|
|
||||||
time.Sleep(time.Hour)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 1)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestChangeThatDoesNotCountIsNoChange(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
loop := start(t, func(path string) bool {
|
|
||||||
return path == "ipinfo_lite.mmdb"
|
|
||||||
})
|
|
||||||
|
|
||||||
// The wait that starts with the watch ends with a call.
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
loop.changes <- fsnotify.Event{Name: "other.mmdb", Op: fsnotify.Create}
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 1)
|
|
||||||
|
|
||||||
loop.changes <- fsnotify.Event{Name: "ipinfo_lite.mmdb", Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 2)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestErrorsOfTheWatchHandedOn(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
loop := start(t, everyPath)
|
|
||||||
|
|
||||||
loop.errs <- errWatch
|
|
||||||
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
if !errors.Is(loop.failure, errWatch) {
|
|
||||||
t.Errorf("failed with %v, want %v", loop.failure, errWatch)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// loop is a callWhenQuiet that a test runs.
|
|
||||||
type loop struct {
|
|
||||||
// changes and errs send it changes and errors.
|
|
||||||
changes chan fsnotify.Event
|
|
||||||
errs chan error
|
|
||||||
// calls counts its calls of changed, and failure is the last error it
|
|
||||||
// handed to failed.
|
|
||||||
calls atomic.Int32
|
|
||||||
failure error
|
|
||||||
}
|
|
||||||
|
|
||||||
// start runs callWhenQuiet, with counts, until the test ends.
|
|
||||||
func start(t *testing.T, counts func(path string) bool) *loop {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
l := &loop{changes: make(chan fsnotify.Event), errs: make(chan error)}
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
callWhenQuiet(ctx, l.changes, l.errs, counts,
|
|
||||||
func() { l.calls.Add(1) },
|
|
||||||
func(err error) { l.failure = err },
|
|
||||||
)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
stop()
|
|
||||||
<-stopped
|
|
||||||
})
|
|
||||||
|
|
||||||
return l
|
|
||||||
}
|
|
||||||
|
|
||||||
// everyPath counts a change to any path.
|
|
||||||
func everyPath(string) bool {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// wantCalls checks how many times changed has been called.
|
|
||||||
func (l *loop) wantCalls(t *testing.T, want int32) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got := l.calls.Load()
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("changed called %d times, want %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+24
-14
@@ -84,7 +84,12 @@ type Params struct {
|
|||||||
// one request to GeoJS is under way at a time, and it asks about every
|
// one request to GeoJS is under way at a time, and it asks about every
|
||||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||||
type GeoJS struct {
|
type GeoJS struct {
|
||||||
params Params
|
url string
|
||||||
|
answered func(Answer)
|
||||||
|
now func() time.Time
|
||||||
|
asked func()
|
||||||
|
failed func(err error, askingAgainIn time.Duration)
|
||||||
|
wentWithoutAnswer func()
|
||||||
// httpClient follows no redirect, so that visitors' addresses go to
|
// httpClient follows no redirect, so that visitors' addresses go to
|
||||||
// GeoJS alone: a redirect is a failure.
|
// GeoJS alone: a redirect is a failure.
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
@@ -140,7 +145,12 @@ func New(params Params) *GeoJS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &GeoJS{
|
return &GeoJS{
|
||||||
params: params,
|
url: params.URL,
|
||||||
|
answered: params.Answered,
|
||||||
|
now: params.Now,
|
||||||
|
asked: params.Asked,
|
||||||
|
failed: params.Failed,
|
||||||
|
wentWithoutAnswer: params.WentWithoutAnswer,
|
||||||
httpClient: &http.Client{
|
httpClient: &http.Client{
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
@@ -183,7 +193,7 @@ func (g *GeoJS) LookUp(
|
|||||||
|
|
||||||
answer, found := g.kept(client)
|
answer, found := g.kept(client)
|
||||||
if !found {
|
if !found {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
}
|
}
|
||||||
|
|
||||||
w, waiting := g.waiting[client]
|
w, waiting := g.waiting[client]
|
||||||
@@ -263,7 +273,7 @@ func (g *GeoJS) Load(answers []Answer) {
|
|||||||
|
|
||||||
g.answers.Purge()
|
g.answers.Purge()
|
||||||
|
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
|
|
||||||
for _, answer := range answers {
|
for _, answer := range answers {
|
||||||
if now.Sub(answer.Answered) < keepFor {
|
if now.Sub(answer.Answered) < keepFor {
|
||||||
@@ -300,7 +310,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w == nil {
|
if w == nil {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil // too many clients wait already
|
return Answer{}, nil // too many clients wait already
|
||||||
}
|
}
|
||||||
@@ -312,7 +322,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w.late {
|
if w.late {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil
|
return Answer{}, nil
|
||||||
}
|
}
|
||||||
@@ -323,7 +333,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
||||||
// and notes that it was used.
|
// and notes that it was used.
|
||||||
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
|
|
||||||
kept, found := g.answers.Get(client)
|
kept, found := g.answers.Get(client)
|
||||||
if !found || now.Sub(kept.Answered) >= keepFor {
|
if !found || now.Sub(kept.Answered) >= keepFor {
|
||||||
@@ -340,7 +350,7 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
// been called. The requests to GeoJS are for every client waiting, so they
|
// been called. The requests to GeoJS are for every client waiting, so they
|
||||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
|
if g.asking || g.stopped || g.now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -372,9 +382,9 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.params.Answered != nil {
|
if g.answered != nil {
|
||||||
for _, answer := range kept {
|
for _, answer := range kept {
|
||||||
g.params.Answered(answer)
|
g.answered(answer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -423,7 +433,7 @@ func (g *GeoJS) keep(
|
|||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
kept := make([]Answer, 0, len(clients))
|
kept := make([]Answer, 0, len(clients))
|
||||||
leftOut := 0
|
leftOut := 0
|
||||||
|
|
||||||
@@ -467,7 +477,7 @@ func (g *GeoJS) keep(
|
|||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
g.params.Failed(err, g.retryDelay)
|
g.failed(err, g.retryDelay)
|
||||||
|
|
||||||
return kept, false
|
return kept, false
|
||||||
}
|
}
|
||||||
@@ -497,7 +507,7 @@ func (g *GeoJS) request(
|
|||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
g.params.URL,
|
g.url,
|
||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -506,7 +516,7 @@ func (g *GeoJS) request(
|
|||||||
|
|
||||||
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
||||||
|
|
||||||
g.params.Asked()
|
g.asked()
|
||||||
|
|
||||||
res, err := g.httpClient.Do(req)
|
res, err := g.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+49
-18
@@ -13,13 +13,18 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
"github.com/oschwald/maxminddb-golang/v2"
|
"github.com/oschwald/maxminddb-golang/v2"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// quietTime is how long the lookup database must go without a change
|
||||||
|
// before it is read again, so that a file still being copied in is read
|
||||||
|
// only once whole.
|
||||||
|
const quietTime = 2 * time.Second
|
||||||
|
|
||||||
// FileParams are what OpenFile needs.
|
// FileParams are what OpenFile needs.
|
||||||
type FileParams struct {
|
type FileParams struct {
|
||||||
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
||||||
@@ -117,13 +122,20 @@ func (f *File) ReadFailures() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Watch watches the directory of the lookup database until ctx is done,
|
// Watch watches the directory of the lookup database until ctx is done,
|
||||||
// and reads the file again once it has gone without a change for a short
|
// and reads the file again once it has gone without a change for
|
||||||
// time, after it is replaced, written or removed, and after Watch starts
|
// quietTime, after it is replaced, written or removed, and after Watch
|
||||||
// watching, as dirwatch describes. A change to another file in its
|
// starts watching. If the directory cannot be watched, that is logged, and
|
||||||
// directory does not count. If the directory cannot be watched, that is
|
// the database read at start stays in use.
|
||||||
// logged, and the database read at start stays in use.
|
|
||||||
func (f *File) Watch(ctx context.Context) {
|
func (f *File) Watch(ctx context.Context) {
|
||||||
watcher, err := dirwatch.New(filepath.Dir(f.params.Path))
|
watcher, err := fsnotify.NewWatcher()
|
||||||
|
if err == nil {
|
||||||
|
defer func() {
|
||||||
|
_ = watcher.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
err = watcher.Add(filepath.Dir(f.params.Path))
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.params.ProcessLog.Error(
|
f.params.ProcessLog.Error(
|
||||||
"cannot watch the lookup database for replacements",
|
"cannot watch the lookup database for replacements",
|
||||||
@@ -134,22 +146,41 @@ func (f *File) Watch(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = watcher.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
||||||
"file", f.params.Path)
|
"file", f.params.Path)
|
||||||
|
|
||||||
watcher.CallWhenQuiet(ctx, f.isDatabase, f.readAgain, func(err error) {
|
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
||||||
f.params.ProcessLog.Warn("watching the lookup database failed",
|
|
||||||
"error", err.Error())
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// isDatabase reports whether path names the lookup database.
|
// readAfterChanges reads the lookup database again once quietTime has
|
||||||
func (f *File) isDatabase(path string) bool {
|
// passed without a change to it from events, until ctx is done, and logs
|
||||||
return filepath.Clean(path) == filepath.Clean(f.params.Path)
|
// the errors from errs. A change to another file in its directory does not
|
||||||
|
// count. The wait starts at once, as if for a change, so that a file
|
||||||
|
// replaced after OpenFile read it, and before its directory was watched,
|
||||||
|
// is read too.
|
||||||
|
func (f *File) readAfterChanges(
|
||||||
|
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
||||||
|
) {
|
||||||
|
path := filepath.Clean(f.params.Path)
|
||||||
|
|
||||||
|
quiet := time.NewTimer(quietTime)
|
||||||
|
defer quiet.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case event := <-events:
|
||||||
|
if filepath.Clean(event.Name) == path {
|
||||||
|
quiet.Reset(quietTime)
|
||||||
|
}
|
||||||
|
case <-quiet.C:
|
||||||
|
f.readAgain()
|
||||||
|
case err := <-errs:
|
||||||
|
f.params.ProcessLog.Warn("watching the lookup database failed",
|
||||||
|
"error", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAgain reads the lookup database again, in place of the one in use,
|
// readAgain reads the lookup database again, in place of the one in use,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package lookup
|
package lookup
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -11,6 +12,7 @@ import (
|
|||||||
"testing/synctest"
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
"github.com/maxmind/mmdbwriter/mmdbtype"
|
"github.com/maxmind/mmdbwriter/mmdbtype"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
@@ -131,48 +133,66 @@ func TestFileThatCannotBeReadIsAnError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
// The tests below run readAfterChanges in a synctest bubble, where time is
|
||||||
t.Parallel()
|
// a clock of the test's own: time.Sleep moves it on at once, and
|
||||||
|
// synctest.Wait returns once readAfterChanges waits again, so that every
|
||||||
|
// reading due by then is done. The test sends the changes itself, as the
|
||||||
|
// watch of a directory cannot run in a bubble.
|
||||||
|
|
||||||
dir := t.TempDir()
|
func TestReplacementCopiedOverTheFileInTwoPartsIsReadOnlyWhole(
|
||||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
t *testing.T,
|
||||||
writeDatabase(t, path, "DE")
|
) {
|
||||||
|
|
||||||
f := openFile(t, path, newQueue())
|
|
||||||
|
|
||||||
for changed, want := range map[string]bool{
|
|
||||||
path: true,
|
|
||||||
dir + "/./ipinfo_lite.mmdb": true,
|
|
||||||
filepath.Join(dir, "other.mmdb"): false,
|
|
||||||
} {
|
|
||||||
if f.isDatabase(changed) != want {
|
|
||||||
t.Errorf(
|
|
||||||
"a change to %s counts: %t, want %t",
|
|
||||||
changed,
|
|
||||||
!want,
|
|
||||||
want,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The tests below run in a synctest bubble, where time is a clock of the
|
|
||||||
// test's own, so that the time a file is read can be checked. The wait for
|
|
||||||
// the directory to go quiet before a reading is tested in dirwatch.
|
|
||||||
|
|
||||||
func TestReplacementIsReadAgain(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||||
writeDatabase(t, path, "DE")
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
queue := newQueue()
|
queue := newQueue()
|
||||||
f := openFile(t, path, queue)
|
f := openFile(t, path, queue)
|
||||||
|
changes := watch(t, f)
|
||||||
|
|
||||||
writeDatabase(t, path, "KP")
|
other := filepath.Join(dir, "replacement.mmdb")
|
||||||
time.Sleep(time.Hour)
|
writeDatabase(t, other, "KP")
|
||||||
f.readAgain()
|
|
||||||
|
//nolint:gosec // a file the test wrote
|
||||||
|
replacement, err := os.ReadFile(other)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", other, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The file in use is overwritten in place, and keeps giving what
|
||||||
|
// it gave. Its first part alone is not a .mmdb file.
|
||||||
|
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create %s: %v", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = file.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
half := len(replacement) / 2
|
||||||
|
write(t, file, replacement[:half])
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
|
// The second part starts the wait again.
|
||||||
|
write(t, file, replacement[half:])
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
wantCountry(t, f, "KP")
|
wantCountry(t, f, "KP")
|
||||||
|
|
||||||
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
||||||
@@ -199,10 +219,15 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
queue := newQueue()
|
queue := newQueue()
|
||||||
f := openFile(t, path, queue)
|
f := openFile(t, path, queue)
|
||||||
read := f.LastRead()
|
read := f.LastRead()
|
||||||
|
changes := watch(t, f)
|
||||||
|
|
||||||
writeFile(t, path, "not a lookup database\n")
|
writeFile(t, path, "not a lookup database\n")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
// Long after, the replacement has been read once.
|
||||||
time.Sleep(time.Hour)
|
time.Sleep(time.Hour)
|
||||||
f.readAgain()
|
synctest.Wait()
|
||||||
wantCountry(t, f, "DE")
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
||||||
@@ -215,7 +240,7 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
}
|
}
|
||||||
|
|
||||||
wantAlerts(t, queue, alerts.Alert{
|
wantAlerts(t, queue, alerts.Alert{
|
||||||
Time: read.Add(time.Hour),
|
Time: read.Add(quietTime),
|
||||||
Event: alerts.EventFileError,
|
Event: alerts.EventFileError,
|
||||||
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
||||||
Detail: map[string]any{
|
Detail: map[string]any{
|
||||||
@@ -227,6 +252,55 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||||
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
|
f := openFile(t, path, newQueue())
|
||||||
|
changes := watch(t, f)
|
||||||
|
|
||||||
|
// The wait that starts with the watch ends with a reading.
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
writeDatabase(t, path, "KP")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: filepath.Join(dir, "other.mmdb"), Op: fsnotify.Create}
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "KP")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplacementSavedBeforeTheWatchStartsIsRead(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||||
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
|
f := openFile(t, path, newQueue())
|
||||||
|
|
||||||
|
// Saved after OpenFile read the file, and before its directory was
|
||||||
|
// watched, so that no change is seen for it.
|
||||||
|
writeDatabase(t, path, "KP")
|
||||||
|
watch(t, f)
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "KP")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// newQueue returns a queue of alerts for a webhook that is never sent
|
// newQueue returns a queue of alerts for a webhook that is never sent
|
||||||
// them, so that they wait in it for the test to look at.
|
// them, so that they wait in it for the test to look at.
|
||||||
func newQueue() *alerts.Queue {
|
func newQueue() *alerts.Queue {
|
||||||
@@ -270,6 +344,28 @@ func openFile(t *testing.T, path string, queue *alerts.Queue) *File {
|
|||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// watch runs f's readAfterChanges until the test ends, and returns the
|
||||||
|
// channel that sends it changes.
|
||||||
|
func watch(t *testing.T, f *File) chan<- fsnotify.Event {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
changes := make(chan fsnotify.Event)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
f.readAfterChanges(ctx, changes, nil)
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
stop()
|
||||||
|
<-stopped
|
||||||
|
})
|
||||||
|
|
||||||
|
return changes
|
||||||
|
}
|
||||||
|
|
||||||
// wantCountry checks the country f gives testClient.
|
// wantCountry checks the country f gives testClient.
|
||||||
func wantCountry(t *testing.T, f *File, want string) {
|
func wantCountry(t *testing.T, f *File, want string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -305,3 +401,13 @@ func writeFile(t *testing.T, path, content string) {
|
|||||||
t.Fatalf("write %s: %v", path, err)
|
t.Fatalf("write %s: %v", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// write writes data to the end of file.
|
||||||
|
func write(t *testing.T, file *os.File, data []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, err := file.Write(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
// Package processlog builds the logger for the process's own messages,
|
|
||||||
// which it writes as JSON lines marked "type":"process". The request log
|
|
||||||
// is package requestlog's.
|
|
||||||
package processlog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Params are what New needs.
|
|
||||||
type Params struct {
|
|
||||||
// Output is where the lines are written.
|
|
||||||
Output io.Writer
|
|
||||||
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
|
|
||||||
InstanceName string
|
|
||||||
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
|
|
||||||
Level slog.Level
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns the logger for the process's own messages: JSON lines on
|
|
||||||
// params.Output, marked "type":"process", with the time in the same form
|
|
||||||
// as a request line's. It writes only the messages at params.Level or more
|
|
||||||
// severe; the request lines requestlog.Write writes are never held back.
|
|
||||||
func New(params Params) *slog.Logger {
|
|
||||||
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
|
|
||||||
Level: params.Level,
|
|
||||||
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
|
||||||
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
|
||||||
return slog.String(
|
|
||||||
slog.TimeKey,
|
|
||||||
requestlog.FormatTime(attr.Value.Time()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return attr
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
return slog.New(handler).
|
|
||||||
With("type", "process", "instance", params.InstanceName)
|
|
||||||
}
|
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
package processlog_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// instanceName is the instance the tests' loggers are built with.
|
|
||||||
const instanceName = "host1/blog"
|
|
||||||
|
|
||||||
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processlog.New(processlog.Params{
|
|
||||||
Output: &out,
|
|
||||||
InstanceName: instanceName,
|
|
||||||
Level: slog.LevelInfo,
|
|
||||||
}).
|
|
||||||
Info("starting",
|
|
||||||
"version", "v1")
|
|
||||||
|
|
||||||
var fields map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(out.Bytes(), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", out.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fields["type"] != "process" ||
|
|
||||||
fields["instance"] != "host1/blog" ||
|
|
||||||
fields["msg"] != "starting" ||
|
|
||||||
fields["level"] != "INFO" ||
|
|
||||||
fields["version"] != "v1" {
|
|
||||||
t.Errorf("process line %v", fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
timeText, _ := fields["time"].(string)
|
|
||||||
|
|
||||||
logged, err := time.Parse(time.RFC3339, timeText)
|
|
||||||
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
|
||||||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
|
||||||
time.Since(logged) > time.Minute {
|
|
||||||
t.Errorf(
|
|
||||||
"process line time %q, want now in UTC with milliseconds",
|
|
||||||
timeText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
levels := []slog.Level{
|
|
||||||
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, level := range levels {
|
|
||||||
t.Run(level.String(), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processLog := processlog.New(processlog.Params{
|
|
||||||
Output: &out,
|
|
||||||
InstanceName: instanceName,
|
|
||||||
Level: level,
|
|
||||||
})
|
|
||||||
for _, at := range levels {
|
|
||||||
processLog.Log(t.Context(), at, "message")
|
|
||||||
}
|
|
||||||
|
|
||||||
var got, want []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(out.String()) {
|
|
||||||
var fields struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", line, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = append(got, fields.Level)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, written := range levels[i:] {
|
|
||||||
want = append(want, written.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("lines at %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
|||||||
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -100,10 +100,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
|||||||
func TestResetKeepsTheHistory(t *testing.T) {
|
func TestResetKeepsTheHistory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerMinute: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -126,7 +123,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
|
|||||||
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
|
|
||||||
for client, requests := range map[string]int{
|
for client, requests := range map[string]int{
|
||||||
"198.51.100.9/32": 2,
|
"198.51.100.9/32": 2,
|
||||||
|
|||||||
@@ -156,28 +156,16 @@ type Request struct {
|
|||||||
BlockedByCoreRuleSet bool
|
BlockedByCoreRuleSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Params are what New needs.
|
// New returns a Limiter for limits, with no client counted yet, whose
|
||||||
type Params struct {
|
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
|
||||||
// Limits are what each client's requests and bytes are counted against.
|
// it, the least recently seen client is dropped, with its history, and
|
||||||
Limits Limits
|
// starts afresh if it comes back.
|
||||||
// MaxClients is the most clients the table holds,
|
func New(limits Limits, maxClients int) *Limiter {
|
||||||
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
|
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
|
||||||
// is dropped, with its history, and starts afresh if it comes back.
|
|
||||||
MaxClients int
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns a Limiter with no client counted yet.
|
|
||||||
func New(params Params) *Limiter {
|
|
||||||
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
|
|
||||||
params.MaxClients,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // NewLRU fails only for a size below one
|
panic(err) // NewLRU fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
limits := params.Limits
|
|
||||||
|
|
||||||
return &Limiter{
|
return &Limiter{
|
||||||
windows: [3]window{
|
windows: [3]window{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
if limiter.OverARateLimit(client, now) {
|
||||||
t.Error("a client never seen is over a rate limit")
|
t.Error("a client never seen is over a rate limit")
|
||||||
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
wantCount(t, limiter, client, now, minute)
|
wantCount(t, limiter, client, now, minute)
|
||||||
|
|
||||||
// A limit that is off is never gone over.
|
// A limit that is off is never gone over.
|
||||||
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
off := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
off.Count(client, now, whole)
|
off.Count(client, now, whole)
|
||||||
|
|
||||||
if off.OverARateLimit(client, now) {
|
if off.OverARateLimit(client, now) {
|
||||||
@@ -95,10 +95,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(tc.limits, tableSize)
|
||||||
Limits: tc.limits,
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
quarter := tc.length / 4
|
quarter := tc.length / 4
|
||||||
@@ -129,10 +126,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -162,13 +159,10 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
|||||||
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{
|
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
|
||||||
PerMinute: 5,
|
tableSize,
|
||||||
BytesPerDay: math.MaxInt64,
|
)
|
||||||
},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -209,10 +203,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Only the hour has limits: the minute's and the day's are off.
|
// Only the hour has limits: the minute's and the day's are off.
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -259,10 +253,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(tc.limits, tableSize)
|
||||||
Limits: tc.limits,
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// 600 bytes are within the limit, 600 more over it.
|
// 600 bytes are within the limit, 600 more over it.
|
||||||
@@ -296,10 +287,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("203.0.113.10/32")
|
other := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -323,7 +314,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
|||||||
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -349,10 +340,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
|
||||||
Limits: ratelimit.Limits{BytesPerDay: 1000},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -368,7 +356,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -423,7 +411,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -450,10 +438,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -475,10 +463,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
|||||||
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -497,10 +482,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
|||||||
func TestRefusedRequestsCount(t *testing.T) {
|
func TestRefusedRequestsCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
refused := netip.MustParsePrefix("203.0.113.9/32")
|
refused := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
within := netip.MustParsePrefix("203.0.113.10/32")
|
within := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -533,10 +518,7 @@ func TestRefusedRequestsCount(t *testing.T) {
|
|||||||
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerMinute: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -552,10 +534,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
|||||||
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -583,10 +562,7 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
|
|||||||
|
|
||||||
const maxClients = 3
|
const maxClients = 3
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
|
||||||
Limits: ratelimit.Limits{PerMinute: 1},
|
|
||||||
MaxClients: maxClients,
|
|
||||||
})
|
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
clients := make([]netip.Prefix, maxClients+1)
|
clients := make([]netip.Prefix, maxClients+1)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
|
|||||||
"2001:db8::/64",
|
"2001:db8::/64",
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
for _, i := range []int{2, 3, 0, 1} {
|
for _, i := range []int{2, 3, 0, 1} {
|
||||||
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
||||||
}
|
}
|
||||||
@@ -52,10 +52,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
before := ratelimit.New(ratelimit.Params{
|
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
for range limit {
|
for range limit {
|
||||||
wantCount(t, before, client, start, "")
|
wantCount(t, before, client, start, "")
|
||||||
}
|
}
|
||||||
@@ -63,10 +60,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
// Loaded into a new limiter, as across a restart, the client has no
|
// Loaded into a new limiter, as across a restart, the client has no
|
||||||
// fresh allowance.
|
// fresh allowance.
|
||||||
later := start.Add(time.Minute)
|
later := start.Add(time.Minute)
|
||||||
after := ratelimit.New(ratelimit.Params{
|
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
after.Load(before.Snapshot(), later)
|
after.Load(before.Snapshot(), later)
|
||||||
wantCount(t, after, client, later, hour)
|
wantCount(t, after, client, later, hour)
|
||||||
}
|
}
|
||||||
@@ -77,7 +71,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
limiter.Count(client, start, whole)
|
limiter.Count(client, start, whole)
|
||||||
limiter.CountBytes(client, start, 5, whole)
|
limiter.CountBytes(client, start, 5, whole)
|
||||||
limiter.CountRefusal(client, start, limit)
|
limiter.CountRefusal(client, start, limit)
|
||||||
@@ -86,7 +80,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
loaded := func(now time.Time) ratelimit.Client {
|
loaded := func(now time.Time) ratelimit.Client {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
after := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
after.Load(limiter.Snapshot(), now)
|
after.Load(limiter.Snapshot(), now)
|
||||||
|
|
||||||
return after.Snapshot()[0]
|
return after.Snapshot()[0]
|
||||||
@@ -148,7 +142,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
|
|||||||
addr = addr.Next()
|
addr = addr.Next()
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
|
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
|
||||||
limiter.Load(clients, midnight())
|
limiter.Load(clients, midnight())
|
||||||
|
|
||||||
got := limiter.Snapshot()
|
got := limiter.Snapshot()
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Package requestlog writes the request log sww prints on stdout: JSON
|
// Package requestlog writes the lines sww prints on stdout: one
|
||||||
// lines marked "type":"request", at most two from waf and a short one
|
// JSON object per request, two from waf and a short one from rproxy,
|
||||||
// from rproxy for each request. Package processlog writes the process's
|
// marked "type":"request", and the process's own messages as JSON lines marked
|
||||||
// own messages.
|
// "type":"process".
|
||||||
package requestlog
|
package requestlog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
@@ -299,3 +300,31 @@ func Milliseconds(d time.Duration) float64 {
|
|||||||
time.Millisecond/time.Microsecond,
|
time.Millisecond/time.Microsecond,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewProcessLogger returns the logger for the process's own messages:
|
||||||
|
// JSON lines on w, marked "type":"process", with the time in the same form
|
||||||
|
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
|
||||||
|
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
|
||||||
|
// the request lines Write writes are never held back.
|
||||||
|
func NewProcessLogger(
|
||||||
|
w io.Writer,
|
||||||
|
instanceName string,
|
||||||
|
level slog.Level,
|
||||||
|
) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
|
||||||
|
Level: level,
|
||||||
|
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
||||||
|
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
||||||
|
return slog.String(
|
||||||
|
slog.TimeKey,
|
||||||
|
FormatTime(attr.Value.Time()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return attr
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return slog.New(handler).
|
||||||
|
With("type", "process", "instance", instanceName)
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@ package requestlog_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -132,3 +134,88 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
|
|||||||
t.Errorf("wrote %s, want %s", out.String(), want)
|
t.Errorf("wrote %s, want %s", out.String(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
|
||||||
|
Info("starting",
|
||||||
|
"version", "v1")
|
||||||
|
|
||||||
|
var fields map[string]any
|
||||||
|
|
||||||
|
err := json.Unmarshal(out.Bytes(), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", out.String(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields["type"] != "process" ||
|
||||||
|
fields["instance"] != "host1/blog" ||
|
||||||
|
fields["msg"] != "starting" ||
|
||||||
|
fields["level"] != "INFO" ||
|
||||||
|
fields["version"] != "v1" {
|
||||||
|
t.Errorf("process line %v", fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeText, _ := fields["time"].(string)
|
||||||
|
|
||||||
|
logged, err := time.Parse(time.RFC3339, timeText)
|
||||||
|
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
||||||
|
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
||||||
|
time.Since(logged) > time.Minute {
|
||||||
|
t.Errorf(
|
||||||
|
"process line time %q, want now in UTC with milliseconds",
|
||||||
|
timeText,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
levels := []slog.Level{
|
||||||
|
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, level := range levels {
|
||||||
|
t.Run(level.String(), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processLog := requestlog.NewProcessLogger(
|
||||||
|
&out,
|
||||||
|
"host1/blog",
|
||||||
|
level,
|
||||||
|
)
|
||||||
|
for _, at := range levels {
|
||||||
|
processLog.Log(t.Context(), at, "message")
|
||||||
|
}
|
||||||
|
|
||||||
|
var got, want []string
|
||||||
|
|
||||||
|
for line := range strings.Lines(out.String()) {
|
||||||
|
var fields struct {
|
||||||
|
Level string `json:"level"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(line), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", line, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got = append(got, fields.Level)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, written := range levels[i:] {
|
||||||
|
want = append(want, written.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("lines at %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: io.Discard,
|
io.Discard,
|
||||||
InstanceName: "test",
|
"test",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
||||||
@@ -870,14 +870,10 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
|
|||||||
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// rproxy is waiting for the rest of the body well within this limit, even on
|
|
||||||
// a busy host, and answers when it runs out, well before dial's deadline.
|
|
||||||
const limit = 5 * time.Second
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
maxRequestTime: "3000",
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1266,9 +1262,7 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// The app's answer arrives well within this limit, even on a busy host, and
|
const limit = 2 * time.Second
|
||||||
// the connection closes when it runs out, well before dial's deadline.
|
|
||||||
const limit = 5 * time.Second
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, answerBeforeTheBody)
|
app := startApp(t, answerBeforeTheBody)
|
||||||
@@ -1433,20 +1427,17 @@ func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(
|
|||||||
|
|
||||||
const answer = "the end of a long answer"
|
const answer = "the end of a long answer"
|
||||||
|
|
||||||
// The request reaches the app well within this limit, even on a busy host.
|
|
||||||
const limit = 5 * time.Second
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
_ = http.NewResponseController(w).Flush()
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
time.Sleep(limit + 100*time.Millisecond)
|
time.Sleep(2100 * time.Millisecond)
|
||||||
|
|
||||||
_, _ = io.WriteString(w, answer)
|
_, _ = io.WriteString(w, answer)
|
||||||
})
|
})
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
maxRequestTime: "2000",
|
||||||
})
|
})
|
||||||
|
|
||||||
got := visit(
|
got := visit(
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -383,11 +383,11 @@ func newServer(
|
|||||||
return rproxy.New(rproxy.Params{
|
return rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: processLog,
|
processLog,
|
||||||
InstanceName: "test",
|
"test",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1452,18 +1452,14 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
|
|||||||
|
|
||||||
greet(w, r)
|
greet(w, r)
|
||||||
})
|
})
|
||||||
out := &output{}
|
addr, out := startRProxy(
|
||||||
server := newServer(
|
t,
|
||||||
t, standIn.URL, app.URL,
|
standIn.URL,
|
||||||
map[string]string{wafBackendTimeout: "200"},
|
app.URL,
|
||||||
out, out,
|
map[string]string{
|
||||||
|
wafBackendTimeout: "200",
|
||||||
|
},
|
||||||
)
|
)
|
||||||
// The breaker's clock stands still, as in
|
|
||||||
// TestAnswersThatCannotBePassedOnOpenTheBreaker: on a slow host, the five
|
|
||||||
// failures could take more than five seconds, and the next request could
|
|
||||||
// come more than a second later.
|
|
||||||
server.StopBreakerClock(time.Now())
|
|
||||||
addr := serve(t, server)
|
|
||||||
|
|
||||||
// An ask can time out before it reaches the stand-in, so these
|
// An ask can time out before it reaches the stand-in, so these
|
||||||
// may leave fewer than five questions, or leave some late.
|
// may leave fewer than five questions, or leave some late.
|
||||||
@@ -1835,15 +1831,11 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// The app's answer to the upgrade arrives well within these limits, even on a
|
|
||||||
// busy host.
|
|
||||||
const limit = 5 * time.Second
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, echoAfterUpgrade)
|
app := startApp(t, echoAfterUpgrade)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
maxRequestTime: "2000",
|
||||||
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
maxResponseTime: "2000",
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1868,10 +1860,7 @@ func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Past both time limits, the connection still carries what is sent.
|
// Past both time limits, the connection still carries what is sent.
|
||||||
time.Sleep(limit + 100*time.Millisecond)
|
time.Sleep(2100 * time.Millisecond)
|
||||||
|
|
||||||
// dial's deadline ran from before the upgrade and this wait.
|
|
||||||
_ = conn.SetDeadline(time.Now().Add(waitLimit))
|
|
||||||
|
|
||||||
const message = "still here\n"
|
const message = "still here\n"
|
||||||
|
|
||||||
|
|||||||
+48
-19
@@ -18,11 +18,12 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
"golang.org/x/net/http/httpguts"
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The actions a rule takes when it matches.
|
// The actions a rule takes when it matches.
|
||||||
@@ -39,6 +40,12 @@ const (
|
|||||||
// extension ends the name of every rule file.
|
// extension ends the name of every rule file.
|
||||||
const extension = ".rules"
|
const extension = ".rules"
|
||||||
|
|
||||||
|
// quietTime is how long SWW_WAF_RULE_FILES_DIR must go without a change before
|
||||||
|
// the rule files are read again, so that a file still being written, such
|
||||||
|
// as one saved in place, appended to or copied in with scp, is read only
|
||||||
|
// once whole.
|
||||||
|
const quietTime = 2 * time.Second
|
||||||
|
|
||||||
// headerTarget starts the target that is one request header,
|
// headerTarget starts the target that is one request header,
|
||||||
// header:<Name>.
|
// header:<Name>.
|
||||||
const headerTarget = "header:"
|
const headerTarget = "header:"
|
||||||
@@ -169,18 +176,26 @@ func (f *Files) Len() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Watch watches Dir until ctx is done, and reads the rule files again
|
// Watch watches Dir until ctx is done, and reads the rule files again
|
||||||
// once Dir has had no change for a short time, after one is edited, added
|
// once Dir has had no change for quietTime, after one is edited, added or
|
||||||
// or removed, and after Watch starts watching, as dirwatch describes. If
|
// removed, and after Watch starts watching. If they then hold an error,
|
||||||
// they then hold an error, the rules stay as they were, the error is
|
// the rules stay as they were, the error is logged with its file and
|
||||||
// logged with its file and line, and the files are read again after the
|
// line, and the files are read again after the next change. If Dir cannot
|
||||||
// next change. If Dir cannot be watched, that is logged, and the rules
|
// be watched, that is logged, and the rules stay as they were loaded.
|
||||||
// stay as they were loaded. While Enabled is false, Watch returns at once.
|
// While Enabled is false, Watch returns at once.
|
||||||
func (f *Files) Watch(ctx context.Context) {
|
func (f *Files) Watch(ctx context.Context) {
|
||||||
if !f.params.Enabled {
|
if !f.params.Enabled {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
watcher, err := dirwatch.New(f.params.Dir)
|
watcher, err := fsnotify.NewWatcher()
|
||||||
|
if err == nil {
|
||||||
|
defer func() {
|
||||||
|
_ = watcher.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
err = watcher.Add(f.params.Dir)
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
||||||
"error", err.Error())
|
"error", err.Error())
|
||||||
@@ -188,22 +203,36 @@ func (f *Files) Watch(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = watcher.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
f.params.ProcessLog.Info("watching the rule files for edits",
|
f.params.ProcessLog.Info("watching the rule files for edits",
|
||||||
"directory", f.params.Dir)
|
"directory", f.params.Dir)
|
||||||
|
|
||||||
// A change to any file in Dir counts.
|
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
||||||
watcher.CallWhenQuiet(ctx,
|
}
|
||||||
func(string) bool { return true },
|
|
||||||
f.readAgain,
|
// readAfterChanges reads the rule files again once quietTime has passed
|
||||||
func(err error) {
|
// without a change from events, until ctx is done, and logs the errors
|
||||||
|
// from errs. The wait starts at once, as if for a change, so that an edit
|
||||||
|
// saved after Load read the files, and before Dir was watched, is taken
|
||||||
|
// in too.
|
||||||
|
func (f *Files) readAfterChanges(
|
||||||
|
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
||||||
|
) {
|
||||||
|
quiet := time.NewTimer(quietTime)
|
||||||
|
defer quiet.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-events:
|
||||||
|
quiet.Reset(quietTime)
|
||||||
|
case <-quiet.C:
|
||||||
|
f.readAgain()
|
||||||
|
case err := <-errs:
|
||||||
f.params.ProcessLog.Warn("watching the rule files failed",
|
f.params.ProcessLog.Warn("watching the rule files failed",
|
||||||
"error", err.Error())
|
"error", err.Error())
|
||||||
},
|
}
|
||||||
)
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAgain reads the rule files again, in place of the rules loaded, or
|
// readAgain reads the rule files again, in place of the rules loaded, or
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
package rules
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The tests below run readAfterChanges in a synctest bubble, where time is
|
||||||
|
// a clock of the test's own: time.Sleep moves it on at once, and
|
||||||
|
// synctest.Wait returns once readAfterChanges waits again, so that every
|
||||||
|
// reading due by then is done. The test sends the changes itself, as the
|
||||||
|
// watch of a directory cannot run in a bubble.
|
||||||
|
|
||||||
|
func TestFileWrittenInTwoPartsTakenInOnlyWhole(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "50-app.rules")
|
||||||
|
writeFile(t, path, "first path block ^/first\n")
|
||||||
|
files := load(t, dir)
|
||||||
|
changes := run(t, files)
|
||||||
|
|
||||||
|
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = file.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
// The first part ends in the middle of a ban rule's regex, which,
|
||||||
|
// read then, would ban every request.
|
||||||
|
write(t, file, "first path block ^/first\nprobe path ban ^/")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantMatched(t, files, "/anything")
|
||||||
|
|
||||||
|
// The second part starts the wait again.
|
||||||
|
write(t, file, `\.env$`+"\n")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantMatched(t, files, "/.env")
|
||||||
|
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantMatched(t, files, "/.env", "probe")
|
||||||
|
wantMatched(t, files, "/anything")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEditSavedBeforeTheWatchStartsTakenIn(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "50-app.rules")
|
||||||
|
writeFile(t, path, "first path block ^/first\n")
|
||||||
|
files := load(t, dir)
|
||||||
|
|
||||||
|
// Saved after Load read the files, and before the directory was
|
||||||
|
// watched, so that no change is seen for it.
|
||||||
|
writeFile(t, path, "first path block ^/edited\n")
|
||||||
|
run(t, files)
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
wantMatched(t, files, "/edited", "first")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// load loads the rules in dir.
|
||||||
|
func load(t *testing.T, dir string) *Files {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
files, err := Load(Params{
|
||||||
|
Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler),
|
||||||
|
Alerts: alerts.New(alerts.Params{}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return files
|
||||||
|
}
|
||||||
|
|
||||||
|
// run runs files' readAfterChanges until the test ends, and returns the
|
||||||
|
// channel that sends it changes.
|
||||||
|
func run(t *testing.T, files *Files) chan<- fsnotify.Event {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
changes := make(chan fsnotify.Event)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
files.readAfterChanges(ctx, changes, nil)
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
stop()
|
||||||
|
<-stopped
|
||||||
|
})
|
||||||
|
|
||||||
|
return changes
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeFile writes content to the file at path.
|
||||||
|
func writeFile(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
err := os.WriteFile(path, []byte(content), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// write writes text to the end of file.
|
||||||
|
func write(t *testing.T, file *os.File, text string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, err := file.WriteString(text)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantMatched checks the ids of the rules that a GET request for path
|
||||||
|
// matches, in order.
|
||||||
|
func wantMatched(t *testing.T, files *Files, path string, want ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
r := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||||
|
"http://app.example"+path, nil)
|
||||||
|
|
||||||
|
matched := files.Match(r)
|
||||||
|
|
||||||
|
got := make([]string, 0, len(matched))
|
||||||
|
for _, rule := range matched {
|
||||||
|
got = append(got, rule.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("%s matched %v, want %v", path, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2452,7 +2452,7 @@ func newParams(dir string) state.Params {
|
|||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}),
|
}),
|
||||||
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
|
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
||||||
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
||||||
Lists: reputation.New(reputation.Params{
|
Lists: reputation.New(reputation.Params{
|
||||||
BlocklistURLs: []string{
|
BlocklistURLs: []string{
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,11 +23,11 @@ import (
|
|||||||
// setting for either.
|
// setting for either.
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
func RunRProxy(ctx context.Context, params Params) int {
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
hostname, _ := os.Hostname() // "" when the host has no name to give
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(
|
||||||
Output: params.Stdout,
|
params.Stdout,
|
||||||
InstanceName: hostname,
|
hostname,
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
})
|
)
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+16
-22
@@ -19,10 +19,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
@@ -43,11 +43,8 @@ const remoteLogStopTimeout = 2 * time.Second
|
|||||||
func RunWAF(ctx context.Context, params Params) int {
|
func RunWAF(ctx context.Context, params Params) int {
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(params.Stdout,
|
||||||
Output: params.Stdout,
|
config.InstanceName(params.Viper), slog.LevelError)
|
||||||
InstanceName: config.InstanceName(params.Viper),
|
|
||||||
Level: slog.LevelError,
|
|
||||||
})
|
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(params.Viper)
|
cfg, err := config.WAFFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -66,11 +63,11 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
stdout = io.MultiWriter(params.Stdout, remote)
|
stdout = io.MultiWriter(params.Stdout, remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
processLog = processlog.New(processlog.Params{
|
processLog = requestlog.NewProcessLogger(
|
||||||
Output: stdout,
|
stdout,
|
||||||
InstanceName: cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
Level: cfg.LogLevel,
|
cfg.LogLevel,
|
||||||
})
|
)
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
stopSending := startSending(ctx, remote, processLog)
|
stopSending := startSending(ctx, remote, processLog)
|
||||||
@@ -242,17 +239,14 @@ func newLookupFile(
|
|||||||
|
|
||||||
// newLimiter returns the table of clients, with the rate and byte limits.
|
// newLimiter returns the table of clients, with the rate and byte limits.
|
||||||
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
||||||
return ratelimit.New(ratelimit.Params{
|
return ratelimit.New(ratelimit.Limits{
|
||||||
Limits: ratelimit.Limits{
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
}, cfg.MaxTrackedClients)
|
||||||
},
|
|
||||||
MaxClients: cfg.MaxTrackedClients,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLedger returns the ledger of the bans, with the settings for them.
|
// newLedger returns the ledger of the bans, with the settings for them.
|
||||||
|
|||||||
@@ -22,31 +22,42 @@ import (
|
|||||||
// an offence, and the request does not wait for the answer. The score is then
|
// an offence, and the request does not wait for the answer. The score is then
|
||||||
// used for each address of the client. Its figures are the score and
|
// used for each address of the client. Its figures are the score and
|
||||||
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
|
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
|
||||||
type abuseIPDB struct{}
|
type abuseIPDB struct {
|
||||||
|
abuseIPDB *reputation.AbuseIPDB
|
||||||
|
// limiter is the table of clients the rate limits count, whose histories
|
||||||
|
// say which clients are offenders.
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
keySet bool // SWW_WAF_ABUSEIPDB_KEY is set
|
||||||
|
minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE
|
||||||
|
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
hitNoter reputationHitNoter
|
||||||
|
}
|
||||||
|
|
||||||
func (abuseIPDB) name() string { return "abuseipdb" }
|
func (abuseIPDB) name() string { return "abuseipdb" }
|
||||||
|
|
||||||
func (abuseIPDB) check(
|
func (a abuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
if rq.h.config.AbuseIPDBKey == "" {
|
if !a.keySet {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
client := clientGroup(rq.h.config, rq.client)
|
client := clientGroup(a.ipv6GroupPrefix, rq.client)
|
||||||
held, _ := rq.h.limiter.Client(client)
|
held, _ := a.limiter.Client(client)
|
||||||
offender := held.History.Offences != ratelimit.Offences{}
|
offender := held.History.Offences != ratelimit.Offences{}
|
||||||
|
|
||||||
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
||||||
if !hit {
|
if !hit {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.abuseIPDBHit = true
|
rq.abuseIPDBHit = true
|
||||||
rq.addTag(config.AbuseIPDBTag)
|
rq.addTag(config.AbuseIPDBTag)
|
||||||
rq.noteHit(
|
a.hitNoter.noteHit(
|
||||||
|
rq,
|
||||||
bans.ReputationHit{
|
bans.ReputationHit{
|
||||||
Source: reputation.AbuseIPDBSource,
|
Source: reputation.AbuseIPDBSource,
|
||||||
Score: &score,
|
Score: &score,
|
||||||
@@ -54,14 +65,14 @@ func (abuseIPDB) check(
|
|||||||
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
||||||
)
|
)
|
||||||
|
|
||||||
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
if a.action != config.ReputationActionDeny {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientDenied,
|
action: requestlog.ActionClientDenied,
|
||||||
figures: map[string]any{
|
figures: map[string]any{
|
||||||
figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore,
|
figureScore: score, "min_score": a.minScore,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -26,7 +26,9 @@ func BanAbuseIPDBBlacklist(
|
|||||||
scores := make(map[netip.Prefix]int64, len(listed))
|
scores := make(map[netip.Prefix]int64, len(listed))
|
||||||
|
|
||||||
for _, client := range listed {
|
for _, client := range listed {
|
||||||
netblock := banNetblock(cfg, client.Addr)
|
netblock := banNetblock(
|
||||||
|
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr,
|
||||||
|
)
|
||||||
scores[netblock] = max(scores[netblock], client.Score)
|
scores[netblock] = max(scores[netblock], client.Score)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -224,7 +224,11 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
|
|||||||
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
|
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
|
||||||
}
|
}
|
||||||
|
|
||||||
return banNetblock(h.config, addr), nil
|
return banNetblock(
|
||||||
|
h.config.BanScopeV4Prefix,
|
||||||
|
h.config.IPv6GroupPrefix,
|
||||||
|
addr,
|
||||||
|
), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// expiry returns when a ban made at now for duration ends: duration
|
// expiry returns when a ban made at now for duration ends: duration
|
||||||
@@ -295,7 +299,9 @@ func (rq *request) showClient() {
|
|||||||
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
||||||
}
|
}
|
||||||
|
|
||||||
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
|
client, seen := rq.h.limiter.Client(
|
||||||
|
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
|
||||||
|
)
|
||||||
if seen {
|
if seen {
|
||||||
answer.Client = &client
|
answer.Client = &client
|
||||||
}
|
}
|
||||||
|
|||||||
+105
-61
@@ -1,7 +1,7 @@
|
|||||||
package waf
|
package waf
|
||||||
|
|
||||||
// The heuristic active_ban, and the code that makes bans, which several
|
// The heuristic active_ban, and banMaker, which makes the bans of several
|
||||||
// heuristics share. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
|
// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
@@ -25,18 +26,22 @@ func (rq *request) banResponse(action string) *refusal {
|
|||||||
// in, in force at now. It notes for the log line when that ban ends. A request
|
// in, in force at now. It notes for the log line when that ban ends. A request
|
||||||
// that makes the ban permanent, or in observe mode would have, raises the alert
|
// that makes the ban permanent, or in observe mode would have, raises the alert
|
||||||
// for it. Its figures are the ban's netblock and when it ends.
|
// for it. Its figures are the ban's netblock and when it ends.
|
||||||
type activeBan struct{}
|
type activeBan struct {
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (activeBan) name() string { return "active_ban" }
|
func (activeBan) name() string { return "active_ban" }
|
||||||
|
|
||||||
func (activeBan) check(
|
func (a activeBan) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
check := rq.h.ledger.Check
|
ledger := a.banMaker.ledger
|
||||||
if rq.h.config.Observe {
|
|
||||||
check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
|
check := ledger.Check
|
||||||
|
if a.banMaker.observe {
|
||||||
|
check = ledger.Find // the ban refuses nothing, and stays as it is
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, banned, madePermanent := check(rq.client, now)
|
ban, banned, madePermanent := check(rq.client, now)
|
||||||
@@ -46,7 +51,7 @@ func (activeBan) check(
|
|||||||
|
|
||||||
if madePermanent {
|
if madePermanent {
|
||||||
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
|
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
|
||||||
rq.alertBan(ban)
|
a.banMaker.alertBan(rq, ban)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !banned {
|
if !banned {
|
||||||
@@ -81,11 +86,12 @@ func limitFigures(hit ratelimit.Hit) map[string]any {
|
|||||||
|
|
||||||
// countedBytes returns the request's bytes, once it has ended, as the
|
// countedBytes returns the request's bytes, once it has ended, as the
|
||||||
// byte limits and the anomaly thresholds count them: the response's body
|
// byte limits and the anomaly thresholds count them: the response's body
|
||||||
// bytes, the request's, or both, as SWW_WAF_BYTES_COUNTED says.
|
// bytes, the request's, or both, as bytesCounted, SWW_WAF_BYTES_COUNTED,
|
||||||
func (rq *request) countedBytes() int64 {
|
// says.
|
||||||
|
func (rq *request) countedBytes(bytesCounted config.BytesCounted) int64 {
|
||||||
response, request := rq.out.bytes, rq.requestBytes
|
response, request := rq.out.bytes, rq.requestBytes
|
||||||
|
|
||||||
switch rq.h.config.BytesCounted {
|
switch bytesCounted {
|
||||||
case config.BytesCountedResponse:
|
case config.BytesCountedResponse:
|
||||||
return response
|
return response
|
||||||
case config.BytesCountedRequest:
|
case config.BytesCountedRequest:
|
||||||
@@ -96,7 +102,29 @@ func (rq *request) countedBytes() int64 {
|
|||||||
return response + request
|
return response + request
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForLimit bans the client's netblock at now for a broken limit, the
|
// banMaker makes the bans of the heuristics that ban a client, in the ban
|
||||||
|
// ledger, and raises the alert for each. In observe mode it makes no ban, and
|
||||||
|
// raises the alert for the ban it would have made.
|
||||||
|
type banMaker struct {
|
||||||
|
ledger *bans.Ledger
|
||||||
|
// limiter is the table of clients the rate limits count. A ban for a limit
|
||||||
|
// sets the client's counters back to zero, and a ban's notes give how many
|
||||||
|
// requests its netblock has sent.
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
metrics *metrics.Metrics
|
||||||
|
alerts *alerts.Queue
|
||||||
|
// observe is true in observe mode (SWW_WAF_MODE).
|
||||||
|
observe bool
|
||||||
|
// banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a
|
||||||
|
// clear sign of attack give.
|
||||||
|
banResponse int
|
||||||
|
// banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix
|
||||||
|
// SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers.
|
||||||
|
banScopeV4Prefix int
|
||||||
|
ipv6GroupPrefix int
|
||||||
|
}
|
||||||
|
|
||||||
|
// banForLimit bans the netblock of rq's client at now for a broken limit, the
|
||||||
// one hit names, notes the offence for the log line and counts the hit in
|
// one hit names, notes the offence for the log line and counts the hit in
|
||||||
// the metrics. status is what the client was sent, or is sent:
|
// the metrics. status is what the client was sent, or is sent:
|
||||||
// SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for
|
// SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for
|
||||||
@@ -105,7 +133,8 @@ func (rq *request) countedBytes() int64 {
|
|||||||
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
|
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
|
||||||
// the client's counters back to zero. In observe mode, banClient says what
|
// the client's counters back to zero. In observe mode, banClient says what
|
||||||
// happens instead.
|
// happens instead.
|
||||||
func (rq *request) banForLimit(
|
func (b banMaker) banForLimit(
|
||||||
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
hit ratelimit.Hit,
|
hit ratelimit.Hit,
|
||||||
status int,
|
status int,
|
||||||
@@ -120,7 +149,8 @@ func (rq *request) banForLimit(
|
|||||||
}
|
}
|
||||||
|
|
||||||
rq.line.Offence = requestlog.OffenceLimit
|
rq.line.Offence = requestlog.OffenceLimit
|
||||||
rq.h.metrics.LimitHit(hit)
|
|
||||||
|
b.metrics.LimitHit(hit)
|
||||||
|
|
||||||
notes := bans.Notes{
|
notes := bans.Notes{
|
||||||
Kind: hit.Kind,
|
Kind: hit.Kind,
|
||||||
@@ -136,81 +166,94 @@ func (rq *request) banForLimit(
|
|||||||
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
|
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banClient(
|
b.banClient(
|
||||||
|
rq,
|
||||||
now,
|
now,
|
||||||
status,
|
status,
|
||||||
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
|
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForAttack bans the client's netblock at now for a clear sign of
|
// banForAttack bans the netblock of rq's client at now for a clear sign of
|
||||||
// attack, which notes name: the ban rule that matched, or the trap path
|
// attack, which notes name: the ban rule that matched, or the trap path
|
||||||
// asked for. It returns the action for that ban. In observe mode, banClient
|
// asked for. It returns the action for that ban. In observe mode, banClient
|
||||||
// says what happens instead.
|
// says what happens instead.
|
||||||
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
func (b banMaker) banForAttack(
|
||||||
return rq.banClient(
|
rq *request, now time.Time, notes bans.Notes,
|
||||||
|
) string {
|
||||||
|
return b.banClient(
|
||||||
|
rq,
|
||||||
now,
|
now,
|
||||||
rq.h.config.BanResponse,
|
b.banResponse,
|
||||||
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
|
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// banClient bans the client's netblock at now on grounds, and returns the
|
// banClient bans the netblock of rq's client at now on grounds, and returns
|
||||||
// action for that ban. It adds to the notes what the request tells of the
|
// the action for that ban. It adds to the notes what the request tells of the
|
||||||
// client, with status, what the client was sent. A ban for a broken limit
|
// client, with status, what the client was sent. A ban for a broken limit
|
||||||
// sets the client's counters back to zero.
|
// sets the client's counters back to zero.
|
||||||
//
|
//
|
||||||
// In observe mode it makes no ban and sets nothing back. It raises the
|
// In observe mode it makes no ban and sets nothing back. It raises the
|
||||||
// alert for the ban it would have made, if that alert would be sent.
|
// alert for the ban it would have made, if that alert would be sent.
|
||||||
func (rq *request) banClient(
|
func (b banMaker) banClient(
|
||||||
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
status int,
|
status int,
|
||||||
grounds bans.Grounds,
|
grounds bans.Grounds,
|
||||||
) string {
|
) string {
|
||||||
netblock := banNetblock(rq.h.config, rq.client)
|
netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client)
|
||||||
|
|
||||||
if rq.h.config.Observe {
|
if b.observe {
|
||||||
if !rq.wouldAlertBan(netblock, now, grounds.Cause) {
|
if !b.wouldAlertBan(netblock, now, grounds.Cause) {
|
||||||
return banAction(
|
return banAction(
|
||||||
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
|
b.ledger.WouldBePermanent(netblock, now, grounds.Cause),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
grounds.Notes = rq.withClientNotes(
|
grounds.Notes = b.withClientNotes(
|
||||||
|
rq,
|
||||||
grounds.Notes,
|
grounds.Notes,
|
||||||
now,
|
now,
|
||||||
status,
|
status,
|
||||||
netblock,
|
netblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
|
ban, wouldBan := b.ledger.WouldBan(netblock, now, grounds)
|
||||||
if wouldBan {
|
if wouldBan {
|
||||||
rq.alertBan(ban)
|
b.alertBan(rq, ban)
|
||||||
}
|
}
|
||||||
|
|
||||||
return banAction(ban.Permanent())
|
return banAction(ban.Permanent())
|
||||||
}
|
}
|
||||||
|
|
||||||
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock)
|
grounds.Notes = b.withClientNotes(
|
||||||
|
rq,
|
||||||
|
grounds.Notes,
|
||||||
|
now,
|
||||||
|
status,
|
||||||
|
netblock,
|
||||||
|
)
|
||||||
|
|
||||||
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
|
ban, made := b.ledger.Ban(netblock, now, grounds)
|
||||||
if grounds.Cause == bans.CauseLimit {
|
if grounds.Cause == bans.CauseLimit {
|
||||||
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
|
b.limiter.Reset(clientGroup(b.ipv6GroupPrefix, rq.client))
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.line.BanExpires = banExpires(ban)
|
rq.line.BanExpires = banExpires(ban)
|
||||||
|
|
||||||
if made {
|
if made {
|
||||||
rq.alertBan(ban)
|
b.alertBan(rq, ban)
|
||||||
}
|
}
|
||||||
|
|
||||||
return banAction(ban.Permanent())
|
return banAction(ban.Permanent())
|
||||||
}
|
}
|
||||||
|
|
||||||
// withClientNotes is notes with what the request tells of the client
|
// withClientNotes is notes with what rq tells of its client added: its AS,
|
||||||
// added: its AS, country and reputation, the request at now with status,
|
// country and reputation, the request at now with status, and how many
|
||||||
// and how many requests netblock, the client's, has sent.
|
// requests netblock, the client's, has sent.
|
||||||
func (rq *request) withClientNotes(
|
func (b banMaker) withClientNotes(
|
||||||
|
rq *request,
|
||||||
notes bans.Notes,
|
notes bans.Notes,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
status int,
|
status int,
|
||||||
@@ -221,7 +264,7 @@ func (rq *request) withClientNotes(
|
|||||||
notes.Country = rq.line.Country
|
notes.Country = rq.line.Country
|
||||||
notes.Reputation = rq.reputation
|
notes.Reputation = rq.reputation
|
||||||
notes.Request = rq.noted(now, status)
|
notes.Request = rq.noted(now, status)
|
||||||
notes.Requests = rq.netblockRequests(netblock)
|
notes.Requests = b.netblockRequests(netblock)
|
||||||
|
|
||||||
return notes
|
return notes
|
||||||
}
|
}
|
||||||
@@ -231,23 +274,22 @@ func (rq *request) withClientNotes(
|
|||||||
// have made is worked out only then, at most once per
|
// have made is worked out only then, at most once per
|
||||||
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
|
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
|
||||||
// netblock's requests, which can mean going through every client.
|
// netblock's requests, which can mean going through every client.
|
||||||
func (rq *request) wouldAlertBan(
|
func (b banMaker) wouldAlertBan(
|
||||||
netblock netip.Prefix, now time.Time, cause string,
|
netblock netip.Prefix, now time.Time, cause string,
|
||||||
) bool {
|
) bool {
|
||||||
event := alerts.EventBan
|
event := alerts.EventBan
|
||||||
if rq.h.ledger.WouldBePermanent(netblock, now, cause) {
|
if b.ledger.WouldBePermanent(netblock, now, cause) {
|
||||||
event = alerts.EventPermanentBan
|
event = alerts.EventPermanentBan
|
||||||
}
|
}
|
||||||
|
|
||||||
return rq.h.alerts.WouldSend(event, netblock)
|
return b.alerts.WouldSend(event, netblock)
|
||||||
}
|
}
|
||||||
|
|
||||||
// alertBan raises the alert for ban, which the request made, or made
|
// alertBan raises the alert for ban, which rq made, or made permanent:
|
||||||
// permanent: permanent_ban for a permanent ban, ban for another. Its
|
// permanent_ban for a permanent ban, ban for another. Its detail gives the
|
||||||
// detail gives the ban's cause, when it ends, and its notes, and in
|
// ban's cause, when it ends, and its notes, and in observe mode, where ban is
|
||||||
// observe mode, where ban is the ban that would have been made, or made
|
// the ban that would have been made, or made permanent, mode, observe.
|
||||||
// permanent, mode, observe.
|
func (b banMaker) alertBan(rq *request, ban bans.Ban) {
|
||||||
func (rq *request) alertBan(ban bans.Ban) {
|
|
||||||
event := alerts.EventBan
|
event := alerts.EventBan
|
||||||
if ban.Permanent() {
|
if ban.Permanent() {
|
||||||
event = alerts.EventPermanentBan
|
event = alerts.EventPermanentBan
|
||||||
@@ -256,11 +298,11 @@ func (rq *request) alertBan(ban bans.Ban) {
|
|||||||
detail := map[string]any{
|
detail := map[string]any{
|
||||||
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
|
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
|
||||||
}
|
}
|
||||||
if rq.h.config.Observe {
|
if b.observe {
|
||||||
detail["mode"] = "observe"
|
detail["mode"] = "observe"
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.h.alerts.Raise(alerts.Alert{
|
b.alerts.Raise(alerts.Alert{
|
||||||
Event: event,
|
Event: event,
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
@@ -273,6 +315,13 @@ func (rq *request) alertBan(ban bans.Ban) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// netblockRequests is how many requests netblock has sent since it was
|
||||||
|
// first seen, the request being banned for included: the histories count it
|
||||||
|
// only once it has ended.
|
||||||
|
func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
|
||||||
|
return b.limiter.Requests(netblock) + 1
|
||||||
|
}
|
||||||
|
|
||||||
// noted is the request, at now, with status, what the client was sent, or
|
// noted is the request, at now, with status, what the client was sent, or
|
||||||
// in observe mode would have been, as the notes of the ban it makes keep
|
// in observe mode would have been, as the notes of the ban it makes keep
|
||||||
// it.
|
// it.
|
||||||
@@ -287,23 +336,18 @@ func (rq *request) noted(now time.Time, status int) bans.Request {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// netblockRequests is how many requests netblock has sent since it was
|
|
||||||
// first seen, this one included: the histories count it only once it has
|
|
||||||
// ended.
|
|
||||||
func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
|
|
||||||
return rq.h.limiter.Requests(netblock) + 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// banNetblock is the netblock a ban on client covers: its IPv4 address,
|
// banNetblock is the netblock a ban on client covers: its IPv4 address,
|
||||||
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
|
// widened to banScopeV4Prefix, SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group
|
||||||
// counts it in.
|
// clientGroup counts it in, of length ipv6GroupPrefix.
|
||||||
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix {
|
func banNetblock(
|
||||||
|
banScopeV4Prefix, ipv6GroupPrefix int, client netip.Addr,
|
||||||
|
) netip.Prefix {
|
||||||
addr := client.Unmap()
|
addr := client.Unmap()
|
||||||
if addr.Is4() {
|
if addr.Is4() {
|
||||||
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked()
|
return netip.PrefixFrom(addr, banScopeV4Prefix).Masked()
|
||||||
}
|
}
|
||||||
|
|
||||||
return clientGroup(cfg, addr)
|
return clientGroup(ipv6GroupPrefix, addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// banExpires is when ban ends, as the log line gives it: a time, or
|
// banExpires is when ban ends, as the log line gives it: a time, or
|
||||||
|
|||||||
@@ -548,10 +548,6 @@ func (s *sender) requestWithHeader(
|
|||||||
// body is sent as it is after the headers, header holding its
|
// body is sent as it is after the headers, header holding its
|
||||||
// Content-Length or Transfer-Encoding. header may hold several lines,
|
// Content-Length or Transfer-Encoding. header may hold several lines,
|
||||||
// separated by "\r\n". It returns the whole answer.
|
// separated by "\r\n". It returns the whole answer.
|
||||||
//
|
|
||||||
// It waits for the answer as long as it takes, since a busy host can slow
|
|
||||||
// the Core Rule Set past any shorter wait. go test's -timeout ends a test
|
|
||||||
// whose answer never comes.
|
|
||||||
func (s *sender) requestWithBody(
|
func (s *sender) requestWithBody(
|
||||||
method, from, path, header, body string, status int, action string,
|
method, from, path, header, body string, status int, action string,
|
||||||
) (logLine, answer) {
|
) (logLine, answer) {
|
||||||
@@ -570,6 +566,11 @@ func (s *sender) requestWithBody(
|
|||||||
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
|
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
|
||||||
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
|
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
|
||||||
|
|
||||||
|
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
|
||||||
|
if err != nil {
|
||||||
|
s.t.Fatalf("set read deadline: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
var got answer
|
var got answer
|
||||||
|
|
||||||
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func (rq *request) runBesideLookup(
|
|||||||
) *verdict {
|
) *verdict {
|
||||||
var running atomic.Pointer[string]
|
var running atomic.Pointer[string]
|
||||||
|
|
||||||
running.Store(new(trapPaths{}.name()))
|
running.Store(new(rq.h.trapPaths.name()))
|
||||||
|
|
||||||
found := make(chan *besideLookupFindings, 1)
|
found := make(chan *besideLookupFindings, 1)
|
||||||
|
|
||||||
@@ -92,7 +92,7 @@ func (rq *request) runBesideLookup(
|
|||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
|
|
||||||
return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now())
|
return rq.runHeuristics(ctx, rq.h.afterBesideLookup, rq.h.now())
|
||||||
}
|
}
|
||||||
|
|
||||||
// findBesideLookup does the work of the heuristics of besideLookup on the
|
// findBesideLookup does the work of the heuristics of besideLookup on the
|
||||||
@@ -105,39 +105,39 @@ func (rq *request) findBesideLookup(
|
|||||||
) *besideLookupFindings {
|
) *besideLookupFindings {
|
||||||
findings := &besideLookupFindings{}
|
findings := &besideLookupFindings{}
|
||||||
|
|
||||||
findings.trapPath = rq.trapPath()
|
findings.trapPath = rq.h.trapPaths.trapPath(rq)
|
||||||
if findings.trapPath != "" {
|
if findings.trapPath != "" {
|
||||||
findings.refuses = true
|
findings.refuses = true
|
||||||
|
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
|
|
||||||
running.Store(new(ruleFiles{}.name()))
|
running.Store(new(rq.h.ruleFiles.name()))
|
||||||
|
|
||||||
findings.rules = rq.h.rules.Match(rq.in)
|
findings.rules = rq.h.ruleFiles.rules.Match(rq.in)
|
||||||
if refusedByRule(findings.rules) {
|
if refusedByRule(findings.rules) {
|
||||||
findings.refuses = true
|
findings.refuses = true
|
||||||
|
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
|
|
||||||
running.Store(new(coreRuleSet{}.name()))
|
coreRuleSet := rq.h.coreRuleSet
|
||||||
|
running.Store(new(coreRuleSet.name()))
|
||||||
|
|
||||||
cfg := rq.h.config
|
if coreRuleSet.mode == config.CoreRuleSetModeOff ||
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff ||
|
pathExempt(rq.in, coreRuleSet.exemptPaths) {
|
||||||
pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) {
|
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
// A question's body has been read into memory, so that reading it here
|
// A question's body has been read into memory, so that reading it here
|
||||||
// cannot fail.
|
// cannot fail.
|
||||||
findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect(
|
findings.inspection, findings.read, _ = coreRuleSet.inspector.Inspect(
|
||||||
rq.in, rq.client, rq.in.Body,
|
rq.in, rq.client, rq.in.Body,
|
||||||
)
|
)
|
||||||
findings.inspectionTime = time.Since(start)
|
findings.inspectionTime = time.Since(start)
|
||||||
findings.inspected = true
|
findings.inspected = true
|
||||||
findings.refuses = rq.refusedByCoreRuleSet(findings.inspection)
|
findings.refuses = coreRuleSet.refusedByCoreRuleSet(findings.inspection)
|
||||||
|
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
@@ -151,7 +151,7 @@ func (rq *request) applyFindings(
|
|||||||
) *verdict {
|
) *verdict {
|
||||||
rq.findings = findings
|
rq.findings = findings
|
||||||
|
|
||||||
return rq.runHeuristics(ctx, besideLookup(), rq.h.now())
|
return rq.runHeuristics(ctx, rq.h.besideLookup, rq.h.now())
|
||||||
}
|
}
|
||||||
|
|
||||||
// timedOut refuses the request for the heuristics of besideLookup not having
|
// timedOut refuses the request for the heuristics of besideLookup not having
|
||||||
|
|||||||
+36
-15
@@ -2,6 +2,7 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// whole is the percentage of each limit a client gets when no biased
|
// whole is the percentage of each limit a client gets when no biased
|
||||||
@@ -29,6 +30,25 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
|
|||||||
cfg.ASNLimitPercentURL != ""
|
cfg.ASNLimitPercentURL != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// biasedThresholds are the settings and the lists by which limitPercentages
|
||||||
|
// lowers a client's limits.
|
||||||
|
type biasedThresholds struct {
|
||||||
|
// lists hold the file SWW_WAF_ASN_LIMIT_PERCENT_URL names.
|
||||||
|
lists *reputation.Lists
|
||||||
|
tagLimitPercent map[string]int64 // SWW_WAF_TAG_LIMIT_PERCENT
|
||||||
|
asnBytesLimitPercent map[string]int64 // SWW_WAF_ASN_BYTES_LIMIT_PERCENT
|
||||||
|
countryBytesLimitPercent map[string]int64 // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT
|
||||||
|
unknownCountryLimitPercent int64 // SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT
|
||||||
|
// blocklistAction and blocklistLimitPercent are SWW_WAF_BLOCKLIST_ACTION
|
||||||
|
// and its percentage.
|
||||||
|
blocklistAction config.ReputationAction
|
||||||
|
blocklistLimitPercent int64
|
||||||
|
// dnsblAbuseIPDBAction and dnsblAbuseIPDBLimitPercent are
|
||||||
|
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION and its percentage.
|
||||||
|
dnsblAbuseIPDBAction config.ReputationAction
|
||||||
|
dnsblAbuseIPDBLimitPercent int64
|
||||||
|
}
|
||||||
|
|
||||||
// limitPercentages returns the client's limit percentages, for the rate
|
// limitPercentages returns the client's limit percentages, for the rate
|
||||||
// limits and for the byte limits, by its tags, its AS number and country
|
// limits and for the byte limits, by its tags, its AS number and country
|
||||||
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and
|
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and
|
||||||
@@ -45,37 +65,38 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
|
|||||||
// the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and
|
// the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and
|
||||||
// SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for
|
// SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for
|
||||||
// an AS number or a country they list.
|
// an AS number or a country they list.
|
||||||
func (rq *request) limitPercentages() (percentage, percentage) {
|
func (b biasedThresholds) limitPercentages(
|
||||||
cfg := rq.h.config
|
rq *request,
|
||||||
|
) (percentage, percentage) {
|
||||||
asn, country := rq.line.ASN, rq.line.Country
|
asn, country := rq.line.ASN, rq.line.Country
|
||||||
|
|
||||||
unknown := percentage{percent: whole}
|
unknown := percentage{percent: whole}
|
||||||
if country == "" && !rq.skippedLookupWait {
|
if country == "" && !rq.skippedLookupWait {
|
||||||
unknown = percentage{
|
unknown = percentage{
|
||||||
cfg.UnknownCountryLimitPercent,
|
b.unknownCountryLimitPercent,
|
||||||
config.UnknownCountryLimitPercentSetting,
|
config.UnknownCountryLimitPercentSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fetched := percentage{percent: whole}
|
fetched := percentage{percent: whole}
|
||||||
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
if percent, listed := b.lists.ASNLimitPercent(asn); listed {
|
||||||
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
|
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
|
||||||
}
|
}
|
||||||
|
|
||||||
blocklisted := percentage{percent: whole}
|
blocklisted := percentage{percent: whole}
|
||||||
if rq.blocklisted &&
|
if rq.blocklisted &&
|
||||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
b.blocklistAction == config.ReputationActionLimit {
|
||||||
blocklisted = percentage{
|
blocklisted = percentage{
|
||||||
cfg.BlocklistLimitPercent,
|
b.blocklistLimitPercent,
|
||||||
config.BlocklistActionSetting,
|
config.BlocklistActionSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dnsblAbuseIPDBListed := percentage{percent: whole}
|
dnsblAbuseIPDBListed := percentage{percent: whole}
|
||||||
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
||||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
b.dnsblAbuseIPDBAction == config.ReputationActionLimit {
|
||||||
dnsblAbuseIPDBListed = percentage{
|
dnsblAbuseIPDBListed = percentage{
|
||||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
b.dnsblAbuseIPDBLimitPercent,
|
||||||
config.DNSBLAbuseIPDBActionSetting,
|
config.DNSBLAbuseIPDBActionSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -83,11 +104,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
const tagSetting = config.TagLimitPercentSetting
|
const tagSetting = config.TagLimitPercentSetting
|
||||||
|
|
||||||
asnRequests := lowest(
|
asnRequests := lowest(
|
||||||
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
given(b.tagLimitPercent, asnTag(asn), tagSetting),
|
||||||
fetched,
|
fetched,
|
||||||
)
|
)
|
||||||
countryRequests := given(
|
countryRequests := given(
|
||||||
cfg.TagLimitPercent,
|
b.tagLimitPercent,
|
||||||
countryTag(country),
|
countryTag(country),
|
||||||
tagSetting,
|
tagSetting,
|
||||||
)
|
)
|
||||||
@@ -98,23 +119,23 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
if tag != asnTag(asn) && tag != countryTag(country) {
|
if tag != asnTag(asn) && tag != countryTag(country) {
|
||||||
otherTags = lowest(
|
otherTags = lowest(
|
||||||
otherTags,
|
otherTags,
|
||||||
given(cfg.TagLimitPercent, tag, tagSetting),
|
given(b.tagLimitPercent, tag, tagSetting),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
asnBytes, countryBytes := asnRequests, countryRequests
|
asnBytes, countryBytes := asnRequests, countryRequests
|
||||||
if _, listed := cfg.ASNBytesLimitPercent[asn]; listed {
|
if _, listed := b.asnBytesLimitPercent[asn]; listed {
|
||||||
asnBytes = given(
|
asnBytes = given(
|
||||||
cfg.ASNBytesLimitPercent,
|
b.asnBytesLimitPercent,
|
||||||
asn,
|
asn,
|
||||||
config.ASNBytesLimitPercentSetting,
|
config.ASNBytesLimitPercentSetting,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, listed := cfg.CountryBytesLimitPercent[country]; listed {
|
if _, listed := b.countryBytesLimitPercent[country]; listed {
|
||||||
countryBytes = given(
|
countryBytes = given(
|
||||||
cfg.CountryBytesLimitPercent,
|
b.countryBytesLimitPercent,
|
||||||
country,
|
country,
|
||||||
config.CountryBytesLimitPercentSetting,
|
config.CountryBytesLimitPercentSetting,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -15,20 +16,24 @@ import (
|
|||||||
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
|
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
|
||||||
// lowers the client's limits instead (see limitPercentages), and being log, it
|
// lowers the client's limits instead (see limitPercentages), and being log, it
|
||||||
// does nothing more. Its figure is the blocklists that list the client.
|
// does nothing more. Its figure is the blocklists that list the client.
|
||||||
type blocklists struct{}
|
type blocklists struct {
|
||||||
|
lists *reputation.Lists
|
||||||
|
action config.ReputationAction // SWW_WAF_BLOCKLIST_ACTION
|
||||||
|
hitNoter reputationHitNoter
|
||||||
|
}
|
||||||
|
|
||||||
func (blocklists) name() string { return "blocklists" }
|
func (blocklists) name() string { return "blocklists" }
|
||||||
|
|
||||||
func (blocklists) check(
|
func (b blocklists) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
listedBy := rq.h.lists.ListedBy(rq.client)
|
listedBy := b.lists.ListedBy(rq.client)
|
||||||
rq.blocklisted = len(listedBy) > 0
|
rq.blocklisted = len(listedBy) > 0
|
||||||
rq.noteListed(listedBy, "listed by a blocklist")
|
b.hitNoter.noteListed(rq, listedBy, "listed by a blocklist")
|
||||||
|
|
||||||
if !rq.refusedByBlocklists(listedBy) {
|
if !b.refusedByBlocklists(listedBy) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,7 +46,6 @@ func (blocklists) check(
|
|||||||
// refusedByBlocklists reports whether the heuristic blocklists refuses a
|
// refusedByBlocklists reports whether the heuristic blocklists refuses a
|
||||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||||
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
||||||
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
func (b blocklists) refusedByBlocklists(listedBy []string) bool {
|
||||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
|
return len(listedBy) > 0 && b.action == config.ReputationActionDeny
|
||||||
config.ReputationActionDeny
|
|
||||||
}
|
}
|
||||||
@@ -6,6 +6,9 @@ package waf
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// byteLimits counts the request's bytes, as countedBytes gives them, for the
|
// byteLimits counts the request's bytes, as countedBytes gives them, for the
|
||||||
@@ -16,11 +19,16 @@ import (
|
|||||||
// the client over a byte limit, as its limit percentage for the byte limits
|
// the client over a byte limit, as its limit percentage for the byte limits
|
||||||
// lowers it, break it, which bans the client; the response was passed on whole.
|
// lowers it, break it, which bans the client; the response was passed on whole.
|
||||||
// Its figures are the window, the count and the limit.
|
// Its figures are the window, the count and the limit.
|
||||||
type byteLimits struct{}
|
type byteLimits struct {
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
bytesCounted config.BytesCounted // SWW_WAF_BYTES_COUNTED
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (byteLimits) name() string { return "byte_limits" }
|
func (byteLimits) name() string { return "byte_limits" }
|
||||||
|
|
||||||
func (byteLimits) check(
|
func (b byteLimits) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
@@ -29,10 +37,10 @@ func (byteLimits) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
counts, hit, over := rq.h.limiter.CountBytes(
|
counts, hit, over := b.limiter.CountBytes(
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(b.ipv6GroupPrefix, rq.client),
|
||||||
now,
|
now,
|
||||||
rq.countedBytes(),
|
rq.countedBytes(b.bytesCounted),
|
||||||
rq.bytesLimitPercent.percent,
|
rq.bytesLimitPercent.percent,
|
||||||
)
|
)
|
||||||
rq.line.Counts.MinuteBytes = counts.MinuteBytes
|
rq.line.Counts.MinuteBytes = counts.MinuteBytes
|
||||||
@@ -43,7 +51,7 @@ func (byteLimits) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banForLimit(now, hit, rq.out.status)
|
b.banMaker.banForLimit(rq, now, hit, rq.out.status)
|
||||||
|
|
||||||
return &verdict{figures: limitFigures(hit)}
|
return &verdict{figures: limitFigures(hit)}
|
||||||
}
|
}
|
||||||
@@ -5,8 +5,6 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// peerAddress is the address of the request's TCP peer.
|
// peerAddress is the address of the request's TCP peer.
|
||||||
@@ -45,12 +43,13 @@ func scheme(r *http.Request, isQuestion bool) string {
|
|||||||
|
|
||||||
// clientGroup is the client a request is counted toward: its IPv4
|
// clientGroup is the client a request is counted toward: its IPv4
|
||||||
// address, or its IPv6 group, the netblock its IPv6 address is in of the
|
// address, or its IPv6 group, the netblock its IPv6 address is in of the
|
||||||
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
|
// length ipv6GroupPrefix, SWW_WAF_IPV6_GROUP_PREFIX, a /64 by default, since
|
||||||
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
|
// one abuser usually holds a whole /64. An IPv4 address in IPv6 form counts as
|
||||||
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix {
|
// IPv4.
|
||||||
|
func clientGroup(ipv6GroupPrefix int, addr netip.Addr) netip.Prefix {
|
||||||
addr = addr.Unmap()
|
addr = addr.Unmap()
|
||||||
if addr.Is6() {
|
if addr.Is6() {
|
||||||
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked()
|
return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked()
|
||||||
}
|
}
|
||||||
|
|
||||||
return netip.PrefixFrom(addr, addr.BitLen())
|
return netip.PrefixFrom(addr, addr.BitLen())
|
||||||
|
|||||||
+30
-18
@@ -10,6 +10,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -22,11 +23,22 @@ import (
|
|||||||
// exploitsignature, raises the core_rule_set_match alert, and in block mode
|
// exploitsignature, raises the core_rule_set_match alert, and in block mode
|
||||||
// refuses the request with 403, which is an offence its client's history
|
// refuses the request with 403, which is an offence its client's history
|
||||||
// counts. Its figures are the score and the threshold.
|
// counts. Its figures are the score and the threshold.
|
||||||
type coreRuleSet struct{}
|
type coreRuleSet struct {
|
||||||
|
// inspector is the Core Rule Set, nil while SWW_WAF_CORE_RULE_SET_MODE is
|
||||||
|
// off.
|
||||||
|
inspector CoreRuleSetInspector
|
||||||
|
metrics *metrics.Metrics
|
||||||
|
alerts *alerts.Queue
|
||||||
|
mode config.CoreRuleSetMode // SWW_WAF_CORE_RULE_SET_MODE
|
||||||
|
exemptPaths []string // SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS
|
||||||
|
scoreThreshold int // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD
|
||||||
|
observe bool // SWW_WAF_MODE is observe
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
}
|
||||||
|
|
||||||
func (coreRuleSet) name() string { return "core_rule_set" }
|
func (coreRuleSet) name() string { return "core_rule_set" }
|
||||||
|
|
||||||
func (coreRuleSet) check(
|
func (c coreRuleSet) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
@@ -36,7 +48,6 @@ func (coreRuleSet) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := rq.h.config
|
|
||||||
result := findings.inspection
|
result := findings.inspection
|
||||||
rq.requestBytes = int64(len(findings.read))
|
rq.requestBytes = int64(len(findings.read))
|
||||||
|
|
||||||
@@ -47,18 +58,18 @@ func (coreRuleSet) check(
|
|||||||
rq.line.CoreRuleSetScore = &result.Score
|
rq.line.CoreRuleSetScore = &result.Score
|
||||||
|
|
||||||
for _, id := range result.RuleIDs {
|
for _, id := range result.RuleIDs {
|
||||||
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
|
c.metrics.CoreRuleSetMatched(string(c.mode), id)
|
||||||
}
|
}
|
||||||
|
|
||||||
threshold := cfg.CoreRuleSetScoreThreshold
|
threshold := c.scoreThreshold
|
||||||
if threshold == 0 || result.Score < threshold {
|
if threshold == 0 || result.Score < threshold {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.addTag(config.ExploitSignatureTag)
|
rq.addTag(config.ExploitSignatureTag)
|
||||||
rq.alertCoreRuleSetMatch(result)
|
c.alertCoreRuleSetMatch(rq, result)
|
||||||
|
|
||||||
if !rq.refusedByCoreRuleSet(result) {
|
if !c.refusedByCoreRuleSet(result) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,20 +87,21 @@ func (coreRuleSet) check(
|
|||||||
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
|
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
|
||||||
// Rule Set found result of: whether the score is at or over
|
// Rule Set found result of: whether the score is at or over
|
||||||
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
|
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
|
||||||
func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool {
|
func (c coreRuleSet) refusedByCoreRuleSet(result coreruleset.Result) bool {
|
||||||
cfg := rq.h.config
|
threshold := c.scoreThreshold
|
||||||
threshold := cfg.CoreRuleSetScoreThreshold
|
|
||||||
|
|
||||||
return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock &&
|
return c.mode == config.CoreRuleSetModeBlock &&
|
||||||
threshold != 0 && result.Score >= threshold
|
threshold != 0 && result.Score >= threshold
|
||||||
}
|
}
|
||||||
|
|
||||||
// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request,
|
// alertCoreRuleSetMatch raises the core_rule_set_match alert for rq, which the
|
||||||
// which the Core Rule Set scored at result, at or over
|
// Core Rule Set scored at result, at or over
|
||||||
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
|
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
|
||||||
// score, the method and the path with the query, and, for a request that is
|
// score, the method and the path with the query, and, for a request that is
|
||||||
// not refused for it, the mode: detect, or observe in observe mode.
|
// not refused for it, the mode: detect, or observe in observe mode.
|
||||||
func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
func (c coreRuleSet) alertCoreRuleSetMatch(
|
||||||
|
rq *request, result coreruleset.Result,
|
||||||
|
) {
|
||||||
detail := map[string]any{
|
detail := map[string]any{
|
||||||
"rule_ids": result.RuleIDs,
|
"rule_ids": result.RuleIDs,
|
||||||
"score": result.Score,
|
"score": result.Score,
|
||||||
@@ -98,17 +110,17 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
case c.mode == config.CoreRuleSetModeDetect:
|
||||||
detail["mode"] = string(config.CoreRuleSetModeDetect)
|
detail["mode"] = string(config.CoreRuleSetModeDetect)
|
||||||
case rq.h.config.Observe:
|
case c.observe:
|
||||||
detail["mode"] = "observe"
|
detail["mode"] = "observe"
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.h.alerts.Raise(alerts.Alert{
|
c.alerts.Raise(alerts.Alert{
|
||||||
Event: alerts.EventCoreRuleSetMatch,
|
Event: alerts.EventCoreRuleSetMatch,
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: clientGroup(rq.h.config, rq.client),
|
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.line.ASN,
|
||||||
ASName: rq.line.ASName,
|
ASName: rq.line.ASName,
|
||||||
Country: rq.line.Country,
|
Country: rq.line.Country,
|
||||||
|
|||||||
@@ -15,16 +15,18 @@ import (
|
|||||||
// does not hold the client's country as it was looked up. A client without a
|
// does not hold the client's country as it was looked up. A client without a
|
||||||
// country, or whose country cannot be found, is refused. Its figures are the
|
// country, or whose country cannot be found, is refused. Its figures are the
|
||||||
// country and the setting that refused it.
|
// country and the setting that refused it.
|
||||||
type countries struct{}
|
type countries struct {
|
||||||
|
allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES
|
||||||
|
}
|
||||||
|
|
||||||
func (countries) name() string { return "countries" }
|
func (countries) name() string { return "countries" }
|
||||||
|
|
||||||
func (countries) check(
|
func (c countries) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
allowed := rq.h.config.ExclusivelyAllowedCountries
|
allowed := c.allowed
|
||||||
country := rq.line.Country
|
country := rq.line.Country
|
||||||
|
|
||||||
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"net/netip"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
@@ -11,12 +12,18 @@ import (
|
|||||||
|
|
||||||
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
|
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
|
||||||
// the setting the client is in.
|
// the setting the client is in.
|
||||||
type denyNets struct{}
|
type denyNets struct {
|
||||||
|
netblocks []netip.Prefix // SWW_WAF_DENY_NETS
|
||||||
|
}
|
||||||
|
|
||||||
func (denyNets) name() string { return "deny_nets" }
|
func (denyNets) name() string { return "deny_nets" }
|
||||||
|
|
||||||
func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
func (d denyNets) check(
|
||||||
for _, netblock := range rq.h.config.DenyNets {
|
_ context.Context,
|
||||||
|
rq *request,
|
||||||
|
_ time.Time,
|
||||||
|
) *verdict {
|
||||||
|
for _, netblock := range d.netblocks {
|
||||||
if netblock.Contains(rq.client) {
|
if netblock.Contains(rq.client) {
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientDenied,
|
action: requestlog.ActionClientDenied,
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package waf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/netip"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDenyNetsRefusesOnlyAClientInsideItsNetblocks(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
heuristic := denyNets{
|
||||||
|
netblocks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
|
||||||
|
}
|
||||||
|
|
||||||
|
// The request has no handler: the heuristic reads only what it holds.
|
||||||
|
inside := &request{
|
||||||
|
decision: decision{client: netip.MustParseAddr("198.51.100.7")},
|
||||||
|
}
|
||||||
|
|
||||||
|
v := heuristic.check(t.Context(), inside, time.Time{})
|
||||||
|
if v == nil {
|
||||||
|
t.Fatal("a client inside the netblock was let through")
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.action != requestlog.ActionClientDenied {
|
||||||
|
t.Errorf(
|
||||||
|
"action = %q, want %q",
|
||||||
|
v.action,
|
||||||
|
requestlog.ActionClientDenied,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := v.figures[figureNetblock]; got != "198.51.100.0/24" {
|
||||||
|
t.Errorf("netblock figure = %v, want 198.51.100.0/24", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
outside := &request{
|
||||||
|
decision: decision{client: netip.MustParseAddr("203.0.113.7")},
|
||||||
|
}
|
||||||
|
|
||||||
|
if v := heuristic.check(t.Context(), outside, time.Time{}); v != nil {
|
||||||
|
t.Errorf("a client outside the netblock was refused: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
+14
-6
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -19,21 +20,28 @@ import (
|
|||||||
// being log, it does nothing more. A zone without a verdict on the client is
|
// being log, it does nothing more. A zone without a verdict on the client is
|
||||||
// asked about it in the background, and the request does not wait for the
|
// asked about it in the background, and the request does not wait for the
|
||||||
// answer. Its figure is the zones that list the client, their keys masked.
|
// answer. Its figure is the zones that list the client, their keys masked.
|
||||||
type dnsbl struct{}
|
type dnsbl struct {
|
||||||
|
dnsbl *reputation.DNSBL
|
||||||
|
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
||||||
|
hitNoter reputationHitNoter
|
||||||
|
}
|
||||||
|
|
||||||
func (dnsbl) name() string { return "dnsbl" }
|
func (dnsbl) name() string { return "dnsbl" }
|
||||||
|
|
||||||
func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
func (d dnsbl) check(
|
||||||
listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
|
ctx context.Context,
|
||||||
|
rq *request,
|
||||||
|
_ time.Time,
|
||||||
|
) *verdict {
|
||||||
|
listedBy := d.dnsbl.ListedBy(ctx, rq.client)
|
||||||
rq.dnsblListed = len(listedBy) > 0
|
rq.dnsblListed = len(listedBy) > 0
|
||||||
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
d.hitNoter.noteListed(rq, listedBy, "listed by a DNSBL zone")
|
||||||
|
|
||||||
for _, zone := range listedBy {
|
for _, zone := range listedBy {
|
||||||
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !rq.dnsblListed ||
|
if !rq.dnsblListed || d.action != config.ReputationActionDeny {
|
||||||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,9 +5,10 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// To add a heuristic, write one file with a type that implements heuristic,
|
// To add a heuristic, write one file with a struct that implements heuristic
|
||||||
// and add one line for it to one of the lists below. Its settings go in
|
// and holds the settings and parts it reads. Add one line for it to one of the
|
||||||
// config.WAF, and the comment at the top of its file names them.
|
// lists that newHandler builds, which hands it those settings and parts. Its
|
||||||
|
// settings go in config.WAF, and the comment at the top of its file names them.
|
||||||
|
|
||||||
// heuristic is one of the checks that decide what waf does with a
|
// heuristic is one of the checks that decide what waf does with a
|
||||||
// request.
|
// request.
|
||||||
@@ -36,74 +37,6 @@ const (
|
|||||||
figureNetblock = "netblock"
|
figureNetblock = "netblock"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The lists of heuristics, each in the order its heuristics run. A client in
|
|
||||||
// SWW_WAF_ALLOW_NETS skips them all.
|
|
||||||
|
|
||||||
// beforeLookup are the heuristics that run before the client's AS number and
|
|
||||||
// country are looked up, so that a client they refuse is not looked up.
|
|
||||||
// header_limits comes first: it only measures and counts, and a request it
|
|
||||||
// refuses causes no other work.
|
|
||||||
func beforeLookup() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
headerLimits{},
|
|
||||||
tagLists{},
|
|
||||||
routeWatchTags{},
|
|
||||||
denyNets{},
|
|
||||||
activeBan{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// afterLookup are the heuristics that run once the lookup has been made.
|
|
||||||
func afterLookup() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
countries{},
|
|
||||||
blocklists{},
|
|
||||||
dnsbl{},
|
|
||||||
abuseIPDB{},
|
|
||||||
tagDeny{},
|
|
||||||
rateLimits{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// besideLookup are the heuristics that only read the request. Their work runs
|
|
||||||
// beside the lookup and afterLookup, in findBesideLookup, which changes nothing
|
|
||||||
// of the request. Their check then applies what that work found, and runs only
|
|
||||||
// once waf takes it.
|
|
||||||
func besideLookup() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
trapPaths{},
|
|
||||||
ruleFiles{},
|
|
||||||
coreRuleSet{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// afterBesideLookup are the heuristics that run once afterLookup and
|
|
||||||
// besideLookup have both let the request through. tag_deny runs a second time,
|
|
||||||
// on the tag rule_files and core_rule_set can add.
|
|
||||||
func afterBesideLookup() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
tagDeny{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// afterAnswer are the heuristics that run on a request once waf has
|
|
||||||
// answered it, before its decision line is written. They ban the client, and do
|
|
||||||
// not change what the request got: their verdict's action is not used.
|
|
||||||
func afterAnswer() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
refusalLimit{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// onReport are the heuristics that run on an allowed request when rproxy's
|
|
||||||
// report of it comes, before its report line is written. Like afterAnswer, they
|
|
||||||
// only ban.
|
|
||||||
func onReport() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
byteLimits{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
||||||
// and returns the first verdict, or nil when none gives one. The log line's
|
// and returns the first verdict, or nil when none gives one. The log line's
|
||||||
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ func (rq *request) startLookUp(
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
client := clientGroup(rq.h.config, rq.client)
|
client := clientGroup(rq.h.config.IPv6GroupPrefix, rq.client)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
if rq.h.config.LookupSource == config.LookupSourceFile {
|
||||||
result <- lookupResult{
|
result <- lookupResult{
|
||||||
@@ -143,8 +143,10 @@ func AddLookup(
|
|||||||
) {
|
) {
|
||||||
limiter.AddLookup(answer.Client, answer.Answered,
|
limiter.AddLookup(answer.Client, answer.Answered,
|
||||||
answer.ASN, answer.ASName, answer.Country)
|
answer.ASN, answer.ASName, answer.Country)
|
||||||
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()),
|
netblock := banNetblock(
|
||||||
answer.ASN, answer.ASName, answer.Country)
|
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, answer.Client.Addr(),
|
||||||
|
)
|
||||||
|
ledger.AddLookup(netblock, answer.ASN, answer.ASName, answer.Country)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
|
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
|
||||||
|
|||||||
+168
-31
@@ -133,15 +133,19 @@ type Server struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newHandler returns the handler with what every request shares, the parts
|
// newHandler returns the handler with what every request shares, the parts
|
||||||
// in params among it.
|
// in params among it. It builds the heuristics, each with the settings and
|
||||||
|
// parts it reads, and the lists they run in.
|
||||||
|
//
|
||||||
|
//nolint:funlen // a few lines for each heuristic, a list that grows with them
|
||||||
func newHandler(params Params) *handler {
|
func newHandler(params Params) *handler {
|
||||||
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // New fails only for a size below one
|
panic(err) // New fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
return &handler{
|
cfg := params.Config
|
||||||
config: params.Config,
|
h := &handler{
|
||||||
|
config: cfg,
|
||||||
requestLog: params.RequestLog,
|
requestLog: params.RequestLog,
|
||||||
processLog: params.ProcessLog,
|
processLog: params.ProcessLog,
|
||||||
now: params.Now,
|
now: params.Now,
|
||||||
@@ -152,47 +156,142 @@ func newHandler(params Params) *handler {
|
|||||||
geojs: params.GeoJS,
|
geojs: params.GeoJS,
|
||||||
anomalies: params.Anomalies,
|
anomalies: params.Anomalies,
|
||||||
lookupFile: params.LookupFile,
|
lookupFile: params.LookupFile,
|
||||||
lists: params.Lists,
|
|
||||||
dnsbl: params.DNSBL,
|
|
||||||
abuseIPDB: params.AbuseIPDB,
|
|
||||||
routeWatch: params.RouteWatch,
|
|
||||||
rules: params.Rules,
|
|
||||||
coreRuleSet: params.CoreRuleSet,
|
|
||||||
alerts: params.Alerts,
|
|
||||||
// The country list, the headers, the tag settings that name an asn=
|
// The country list, the headers, the tag settings that name an asn=
|
||||||
// or country= tag and the biased thresholds act on the answer before
|
// or country= tag and the biased thresholds act on the answer before
|
||||||
// the request goes on.
|
// the request goes on.
|
||||||
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
|
||||||
params.Config.AddLookupHeaders ||
|
cfg.AddLookupHeaders ||
|
||||||
params.Config.TagsNeedLookup() ||
|
cfg.TagsNeedLookup() ||
|
||||||
biasedThresholdsSet(params.Config),
|
biasedThresholdsSet(cfg),
|
||||||
besideLookupLimit: cmp.Or(
|
besideLookupLimit: cmp.Or(
|
||||||
params.BesideLookupTimeLimit,
|
params.BesideLookupTimeLimit,
|
||||||
besideLookupTimeLimit,
|
besideLookupTimeLimit,
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sharedBanMaker := banMaker{
|
||||||
|
ledger: params.Ledger,
|
||||||
|
limiter: params.Limiter,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
alerts: params.Alerts,
|
||||||
|
observe: cfg.Observe,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
banScopeV4Prefix: cfg.BanScopeV4Prefix,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
}
|
||||||
|
hitNoter := reputationHitNoter{
|
||||||
|
metrics: params.Metrics,
|
||||||
|
alerts: params.Alerts,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
}
|
||||||
|
|
||||||
|
h.blocklists = blocklists{
|
||||||
|
lists: params.Lists,
|
||||||
|
action: cfg.BlocklistAction,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
}
|
||||||
|
h.rateLimits = rateLimits{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
exemptNets: cfg.RateLimitExemptNets,
|
||||||
|
exemptPaths: cfg.RateLimitExemptPaths,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
biasedThresholds: biasedThresholds{
|
||||||
|
lists: params.Lists,
|
||||||
|
tagLimitPercent: cfg.TagLimitPercent,
|
||||||
|
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
|
||||||
|
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
|
||||||
|
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
|
||||||
|
blocklistAction: cfg.BlocklistAction,
|
||||||
|
blocklistLimitPercent: cfg.BlocklistLimitPercent,
|
||||||
|
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
|
},
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
}
|
||||||
|
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
|
||||||
|
h.ruleFiles = ruleFiles{
|
||||||
|
rules: params.Rules,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
}
|
||||||
|
h.coreRuleSet = coreRuleSet{
|
||||||
|
inspector: params.CoreRuleSet,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
alerts: params.Alerts,
|
||||||
|
mode: cfg.CoreRuleSetMode,
|
||||||
|
exemptPaths: cfg.CoreRuleSetExemptPaths,
|
||||||
|
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
|
||||||
|
observe: cfg.Observe,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
}
|
||||||
|
|
||||||
|
h.beforeLookup = []heuristic{
|
||||||
|
headerLimits{},
|
||||||
|
tagLists{lists: params.Lists},
|
||||||
|
routeWatchTags{
|
||||||
|
routeWatch: params.RouteWatch,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
},
|
||||||
|
denyNets{netblocks: cfg.DenyNets},
|
||||||
|
activeBan{banMaker: sharedBanMaker},
|
||||||
|
}
|
||||||
|
h.afterLookup = []heuristic{
|
||||||
|
countries{allowed: cfg.ExclusivelyAllowedCountries},
|
||||||
|
h.blocklists,
|
||||||
|
dnsbl{
|
||||||
|
dnsbl: params.DNSBL,
|
||||||
|
action: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
},
|
||||||
|
abuseIPDB{
|
||||||
|
abuseIPDB: params.AbuseIPDB,
|
||||||
|
limiter: params.Limiter,
|
||||||
|
keySet: cfg.AbuseIPDBKey != "",
|
||||||
|
minScore: cfg.AbuseIPDBMinScore,
|
||||||
|
action: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
},
|
||||||
|
tagDeny{tags: cfg.TagDeny},
|
||||||
|
h.rateLimits,
|
||||||
|
}
|
||||||
|
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
|
||||||
|
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
|
||||||
|
h.afterAnswer = []heuristic{
|
||||||
|
refusalLimit{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
perMinute: cfg.RefusalsLimitPerMinute,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
h.onReport = []heuristic{
|
||||||
|
byteLimits{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
bytesCounted: cfg.BytesCounted,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return h
|
||||||
}
|
}
|
||||||
|
|
||||||
// handler is waf's handler. It holds what every request shares; what
|
// handler is waf's handler. It holds what every request shares; what
|
||||||
// belongs to one request is in a request.
|
// belongs to one request is in a request.
|
||||||
type handler struct {
|
type handler struct {
|
||||||
config *config.WAF
|
config *config.WAF
|
||||||
requestLog io.Writer
|
requestLog io.Writer
|
||||||
processLog *slog.Logger
|
processLog *slog.Logger
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
metrics *metrics.Metrics
|
metrics *metrics.Metrics
|
||||||
limiter *ratelimit.Limiter
|
limiter *ratelimit.Limiter
|
||||||
ledger *bans.Ledger
|
ledger *bans.Ledger
|
||||||
geojs *geojs.GeoJS
|
geojs *geojs.GeoJS
|
||||||
anomalies *anomaly.Counters
|
anomalies *anomaly.Counters
|
||||||
lookupFile *lookup.File
|
lookupFile *lookup.File
|
||||||
lists *reputation.Lists
|
|
||||||
dnsbl *reputation.DNSBL
|
|
||||||
abuseIPDB *reputation.AbuseIPDB
|
|
||||||
routeWatch *routewatch.Client
|
|
||||||
rules *rules.Files
|
|
||||||
coreRuleSet CoreRuleSetInspector
|
|
||||||
alerts *alerts.Queue
|
|
||||||
// waitForGeoJS is true when a setting needs each request's answer from
|
// waitForGeoJS is true when a setting needs each request's answer from
|
||||||
// GeoJS before the request goes on.
|
// GeoJS before the request goes on.
|
||||||
waitForGeoJS bool
|
waitForGeoJS bool
|
||||||
@@ -202,4 +301,42 @@ type handler struct {
|
|||||||
// awaitingReport holds the decision on each answered question by its id
|
// awaitingReport holds the decision on each answered question by its id
|
||||||
// until its report comes, and nil for a question still being handled.
|
// until its report comes, and nil for a question still being handled.
|
||||||
awaitingReport *lru.Cache[string, *decision]
|
awaitingReport *lru.Cache[string, *decision]
|
||||||
|
|
||||||
|
// The lists of heuristics, each in the order its heuristics run. A client
|
||||||
|
// in SWW_WAF_ALLOW_NETS skips them all.
|
||||||
|
|
||||||
|
// beforeLookup are the heuristics that run before the client's AS number
|
||||||
|
// and country are looked up, so that a client they refuse is not looked
|
||||||
|
// up. header_limits comes first: it only measures and counts, and a
|
||||||
|
// request it refuses causes no other work.
|
||||||
|
beforeLookup []heuristic
|
||||||
|
// afterLookup are the heuristics that run once the lookup has been made.
|
||||||
|
afterLookup []heuristic
|
||||||
|
// besideLookup are the heuristics that only read the request. Their work
|
||||||
|
// runs beside the lookup and afterLookup, in findBesideLookup, which
|
||||||
|
// changes nothing of the request. Their check then applies what that work
|
||||||
|
// found, and runs only once waf takes it.
|
||||||
|
besideLookup []heuristic
|
||||||
|
// afterBesideLookup are the heuristics that run once afterLookup and
|
||||||
|
// besideLookup have both let the request through. tag_deny runs a second
|
||||||
|
// time, on the tag rule_files and core_rule_set can add.
|
||||||
|
afterBesideLookup []heuristic
|
||||||
|
// afterAnswer are the heuristics that run on a request once waf has
|
||||||
|
// answered it, before its decision line is written. They ban the client,
|
||||||
|
// and do not change what the request got: their verdict's action is not
|
||||||
|
// used.
|
||||||
|
afterAnswer []heuristic
|
||||||
|
// onReport are the heuristics that run on an allowed request when
|
||||||
|
// rproxy's report of it comes, before its report line is written. Like
|
||||||
|
// afterAnswer, they only ban.
|
||||||
|
onReport []heuristic
|
||||||
|
|
||||||
|
// The heuristics of the lists that other steps of a request use too:
|
||||||
|
// refusedWhateverTheAnswer asks blocklists and rateLimits, and
|
||||||
|
// findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet.
|
||||||
|
blocklists blocklists
|
||||||
|
rateLimits rateLimits
|
||||||
|
trapPaths trapPaths
|
||||||
|
ruleFiles ruleFiles
|
||||||
|
coreRuleSet coreRuleSet
|
||||||
}
|
}
|
||||||
+24
-24
@@ -24,7 +24,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
@@ -194,11 +193,11 @@ func (o *output) requestLine(t *testing.T) logLine {
|
|||||||
// endedLines waits for count requests that went through rproxy, or to waf's
|
// endedLines waits for count requests that went through rproxy, or to waf's
|
||||||
// own endpoints, to have ended, and returns the last log line of each, in the
|
// own endpoints, to have ended, and returns the last log line of each, in the
|
||||||
// order waf took them. A question's last line is the one its report writes.
|
// order waf took them. A question's last line is the one its report writes.
|
||||||
// It waits as long as it takes, as requestWithBody does.
|
|
||||||
func (o *output) endedLines(t *testing.T, count int) []logLine {
|
func (o *output) endedLines(t *testing.T, count int) []logLine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
for {
|
deadline := time.Now().Add(waitLimit)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
found := o.lastLines(t)
|
found := o.lastLines(t)
|
||||||
|
|
||||||
ended := len(found) >= count
|
ended := len(found) >= count
|
||||||
@@ -214,6 +213,10 @@ func (o *output) endedLines(t *testing.T, count int) []logLine {
|
|||||||
|
|
||||||
time.Sleep(pollInterval)
|
time.Sleep(pollInterval)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// lastLines returns the last request log line written so far of each request,
|
// lastLines returns the last request log line written so far of each request,
|
||||||
@@ -395,11 +398,11 @@ func startRProxy(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: io.Discard,
|
io.Discard,
|
||||||
InstanceName: "rproxy",
|
"rproxy",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
@@ -468,11 +471,11 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
|
|
||||||
out := &output{}
|
out := &output{}
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(
|
||||||
Output: out,
|
out,
|
||||||
InstanceName: cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
Level: cfg.LogLevel,
|
cfg.LogLevel,
|
||||||
})
|
)
|
||||||
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
||||||
@@ -558,17 +561,14 @@ func addCounters(params *waf.Params) {
|
|||||||
cfg.MetricsBusiestCountriesAndASNs,
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
)
|
)
|
||||||
params.Limiter = ratelimit.New(ratelimit.Params{
|
params.Limiter = ratelimit.New(ratelimit.Limits{
|
||||||
Limits: ratelimit.Limits{
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
}, cfg.MaxTrackedClients)
|
||||||
},
|
|
||||||
MaxClients: cfg.MaxTrackedClients,
|
|
||||||
})
|
|
||||||
params.Ledger = bans.New(bans.Params{
|
params.Ledger = bans.New(bans.Params{
|
||||||
LimitBanDuration: cfg.LimitBanDuration,
|
LimitBanDuration: cfg.LimitBanDuration,
|
||||||
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
||||||
|
|||||||
+34
-22
@@ -6,8 +6,10 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"net/netip"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,28 +20,37 @@ import (
|
|||||||
// refused, and bans the client. A request exempt from the rate limits is exempt
|
// refused, and bans the client. A request exempt from the rate limits is exempt
|
||||||
// from the byte limits too. Its figures are the window, the count and the
|
// from the byte limits too. Its figures are the window, the count and the
|
||||||
// limit.
|
// limit.
|
||||||
type rateLimits struct{}
|
type rateLimits struct {
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS
|
||||||
|
exemptPaths []string // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS
|
||||||
|
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
biasedThresholds biasedThresholds
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (rateLimits) name() string { return "rate_limits" }
|
func (rateLimits) name() string { return "rate_limits" }
|
||||||
|
|
||||||
func (rateLimits) check(
|
func (r rateLimits) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
rq.counted = rq.countedByRateLimits()
|
rq.counted = r.countedByRateLimits(rq)
|
||||||
if !rq.counted {
|
if !rq.counted {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.rateLimitPercent, rq.bytesLimitPercent = rq.limitPercentages()
|
rq.rateLimitPercent, rq.bytesLimitPercent =
|
||||||
|
r.biasedThresholds.limitPercentages(rq)
|
||||||
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting =
|
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting =
|
||||||
rq.rateLimitPercent.logged()
|
rq.rateLimitPercent.logged()
|
||||||
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
|
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
|
||||||
rq.bytesLimitPercent.logged()
|
rq.bytesLimitPercent.logged()
|
||||||
|
|
||||||
counts, hit, over := rq.h.limiter.Count(
|
counts, hit, over := r.limiter.Count(
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||||
now,
|
now,
|
||||||
rq.rateLimitPercent.percent,
|
rq.rateLimitPercent.percent,
|
||||||
)
|
)
|
||||||
@@ -49,28 +60,29 @@ func (rateLimits) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banForLimit(now, hit, rq.h.config.BanResponse)
|
r.banMaker.banForLimit(rq, now, hit, r.banResponse)
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
|
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the
|
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq
|
||||||
// request at now, whatever its client's limit percentages: the rate limits
|
// at now, whatever its client's limit percentages: the rate limits count it,
|
||||||
// count it, and one more request from its client takes it over a whole rate
|
// and one more request from its client takes it over a whole rate limit. It
|
||||||
// limit. It counts nothing.
|
// counts nothing.
|
||||||
func (rq *request) overAWholeRateLimit(now time.Time) bool {
|
func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool {
|
||||||
return rq.countedByRateLimits() &&
|
return r.countedByRateLimits(rq) &&
|
||||||
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now)
|
r.limiter.OverARateLimit(
|
||||||
|
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||||
|
now,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// countedByRateLimits reports whether the rate limits count the request: its
|
// countedByRateLimits reports whether the rate limits count rq: its client is
|
||||||
// client is not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
|
// not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt under
|
||||||
// under SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
|
// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
|
||||||
func (rq *request) countedByRateLimits() bool {
|
func (r rateLimits) countedByRateLimits(rq *request) bool {
|
||||||
cfg := rq.h.config
|
return !isInside(rq.client, r.exemptNets) &&
|
||||||
|
!pathExempt(rq.in, r.exemptPaths)
|
||||||
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
|
|
||||||
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
|
|
||||||
}
|
}
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,17 +19,22 @@ import (
|
|||||||
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
|
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
|
||||||
// heuristics skip, is not counted, and nothing is while the limit is off.
|
// heuristics skip, is not counted, and nothing is while the limit is off.
|
||||||
// Its figures are the window, the count and the limit.
|
// Its figures are the window, the count and the limit.
|
||||||
type refusalLimit struct{}
|
type refusalLimit struct {
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE
|
||||||
|
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (refusalLimit) name() string { return "refusal_limit" }
|
func (refusalLimit) name() string { return "refusal_limit" }
|
||||||
|
|
||||||
func (refusalLimit) check(
|
func (r refusalLimit) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
cfg := rq.h.config
|
if r.perMinute == 0 {
|
||||||
if cfg.RefusalsLimitPerMinute == 0 {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,10 +42,10 @@ func (refusalLimit) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
hit, over := rq.h.limiter.CountRefusal(
|
hit, over := r.limiter.CountRefusal(
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||||
now,
|
now,
|
||||||
cfg.RefusalsLimitPerMinute,
|
r.perMinute,
|
||||||
)
|
)
|
||||||
if !over {
|
if !over {
|
||||||
return nil
|
return nil
|
||||||
@@ -54,10 +60,10 @@ func (refusalLimit) check(
|
|||||||
status = http.StatusForbidden
|
status = http.StatusForbidden
|
||||||
case requestlog.ActionTemporarilyBanned,
|
case requestlog.ActionTemporarilyBanned,
|
||||||
requestlog.ActionPermanentlyBanned:
|
requestlog.ActionPermanentlyBanned:
|
||||||
status = cfg.BanResponse
|
status = r.banResponse
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banForLimit(now, hit, status)
|
r.banMaker.banForLimit(rq, now, hit, status)
|
||||||
|
|
||||||
return &verdict{figures: limitFigures(hit)}
|
return &verdict{figures: limitFigures(hit)}
|
||||||
}
|
}
|
||||||
@@ -5,22 +5,35 @@ package waf
|
|||||||
import (
|
import (
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// reputationHitNoter notes the reputation sources that list a request's
|
||||||
|
// client, for the reputation heuristics.
|
||||||
|
type reputationHitNoter struct {
|
||||||
|
metrics *metrics.Metrics
|
||||||
|
alerts *alerts.Queue
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
}
|
||||||
|
|
||||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
// DNSBL zones, their keys masked, that list rq's client, as noteHit does,
|
||||||
// with reason.
|
// with reason.
|
||||||
func (rq *request) noteListed(sources []string, reason string) {
|
func (n reputationHitNoter) noteListed(
|
||||||
|
rq *request, sources []string, reason string,
|
||||||
|
) {
|
||||||
for _, source := range sources {
|
for _, source := range sources {
|
||||||
rq.noteHit(bans.ReputationHit{Source: source}, reason)
|
n.noteHit(rq, bans.ReputationHit{Source: source}, reason)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// noteHit adds hit's source, which lists the client, to the log line's
|
// noteHit adds hit's source, which lists rq's client, to the log line's
|
||||||
// reputation, and hit to the notes of a ban the request makes, counts the
|
// reputation, and hit to the notes of a ban the request makes, counts the
|
||||||
// source in the metrics, and raises a reputation_hit alert with reason,
|
// source in the metrics, and raises a reputation_hit alert with reason,
|
||||||
// whose detail gives hit's source and score.
|
// whose detail gives hit's source and score.
|
||||||
func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
|
func (n reputationHitNoter) noteHit(
|
||||||
|
rq *request, hit bans.ReputationHit, reason string,
|
||||||
|
) {
|
||||||
detail := map[string]any{"source": hit.Source}
|
detail := map[string]any{"source": hit.Source}
|
||||||
if hit.Score != nil {
|
if hit.Score != nil {
|
||||||
detail["score"] = *hit.Score
|
detail["score"] = *hit.Score
|
||||||
@@ -28,12 +41,12 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
|
|||||||
|
|
||||||
rq.line.Reputation = append(rq.line.Reputation, hit.Source)
|
rq.line.Reputation = append(rq.line.Reputation, hit.Source)
|
||||||
rq.reputation = append(rq.reputation, hit)
|
rq.reputation = append(rq.reputation, hit)
|
||||||
rq.h.metrics.ReputationHit(hit.Source)
|
n.metrics.ReputationHit(hit.Source)
|
||||||
rq.h.alerts.Raise(alerts.Alert{
|
n.alerts.Raise(alerts.Alert{
|
||||||
Event: alerts.EventReputationHit,
|
Event: alerts.EventReputationHit,
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: clientGroup(rq.h.config, rq.client),
|
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.line.ASN,
|
||||||
ASName: rq.line.ASName,
|
ASName: rq.line.ASName,
|
||||||
Country: rq.line.Country,
|
Country: rq.line.Country,
|
||||||
|
|||||||
+13
-8
@@ -122,8 +122,11 @@ func (h *handler) startRequest(
|
|||||||
RequestID: requestID(r, isQuestion),
|
RequestID: requestID(r, isQuestion),
|
||||||
PeerIP: peer.String(),
|
PeerIP: peer.String(),
|
||||||
ForwardedFor: strings.Join(forwardedFor, ", "),
|
ForwardedFor: strings.Join(forwardedFor, ", "),
|
||||||
ClientGroup: clientGroup(h.config, client).String(),
|
ClientGroup: clientGroup(
|
||||||
ContentType: r.Header.Get("Content-Type"),
|
h.config.IPv6GroupPrefix,
|
||||||
|
client,
|
||||||
|
).String(),
|
||||||
|
ContentType: r.Header.Get("Content-Type"),
|
||||||
RequestHeaders: requestHeaders(
|
RequestHeaders: requestHeaders(
|
||||||
r,
|
r,
|
||||||
h.config.LogRequestHeaders,
|
h.config.LogRequestHeaders,
|
||||||
@@ -233,7 +236,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict {
|
|||||||
|
|
||||||
now := rq.h.now()
|
now := rq.h.now()
|
||||||
|
|
||||||
v := rq.runHeuristics(ctx, beforeLookup(), now)
|
v := rq.runHeuristics(ctx, rq.h.beforeLookup, now)
|
||||||
if v != nil {
|
if v != nil {
|
||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
@@ -265,7 +268,7 @@ func (rq *request) runAfterLookup(
|
|||||||
// again: a request counted at the time it arrived could be counted
|
// again: a request counted at the time it arrived could be counted
|
||||||
// after a later one of its client's, which the rate limits would take
|
// after a later one of its client's, which the rate limits would take
|
||||||
// for the clock set back.
|
// for the clock set back.
|
||||||
return rq.runHeuristics(ctx, afterLookup(), rq.h.now())
|
return rq.runHeuristics(ctx, rq.h.afterLookup, rq.h.now())
|
||||||
}
|
}
|
||||||
|
|
||||||
// pathExempt reports whether r is exempt under prefixes,
|
// pathExempt reports whether r is exempt under prefixes,
|
||||||
@@ -375,7 +378,7 @@ func (rq *request) addToHistory() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
rq.h.limiter.AddToHistory(
|
rq.h.limiter.AddToHistory(
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
|
||||||
rq.h.now(),
|
rq.h.now(),
|
||||||
request,
|
request,
|
||||||
)
|
)
|
||||||
@@ -404,11 +407,11 @@ func (rq *request) countAnomalies() {
|
|||||||
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
ClientGroup: clientGroup(rq.h.config, rq.client),
|
ClientGroup: clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
|
||||||
ASN: answer.ASN,
|
ASN: answer.ASN,
|
||||||
ASName: answer.ASName,
|
ASName: answer.ASName,
|
||||||
Country: answer.Country,
|
Country: answer.Country,
|
||||||
Bytes: rq.countedBytes(),
|
Bytes: rq.countedBytes(rq.h.config.BytesCounted),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -435,7 +438,9 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
|
|||||||
return rq.lookupAnswer, true
|
return rq.lookupAnswer, true
|
||||||
}
|
}
|
||||||
|
|
||||||
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client))
|
return rq.h.geojs.Kept(
|
||||||
|
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// responseWriter is waf's answer to a request. It notes the status and
|
// responseWriter is waf's answer to a request. It notes the status and
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
)
|
)
|
||||||
|
|
||||||
// routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock>
|
// routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock>
|
||||||
@@ -17,22 +18,26 @@ import (
|
|||||||
// SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and
|
// SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and
|
||||||
// asked for by the request's address, and a route without an origin AS gives only
|
// asked for by the request's address, and a route without an origin AS gives only
|
||||||
// the prefix= tag.
|
// the prefix= tag.
|
||||||
type routeWatchTags struct{}
|
type routeWatchTags struct {
|
||||||
|
// routeWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
|
||||||
|
routeWatch *routewatch.Client
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
}
|
||||||
|
|
||||||
func (routeWatchTags) name() string { return "routewatch_tags" }
|
func (routeWatchTags) name() string { return "routewatch_tags" }
|
||||||
|
|
||||||
func (routeWatchTags) check(
|
func (r routeWatchTags) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
if rq.h.routeWatch == nil {
|
if r.routeWatch == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
route, missing := rq.h.routeWatch.Route(
|
route, missing := r.routeWatch.Route(
|
||||||
ctx,
|
ctx,
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||||
rq.client,
|
rq.client,
|
||||||
)
|
)
|
||||||
if missing != "" {
|
if missing != "" {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
)
|
)
|
||||||
@@ -22,11 +23,15 @@ import (
|
|||||||
// mode raises the alert for the ban
|
// mode raises the alert for the ban
|
||||||
// it would have made. Either rule's match is noted as an offence, for the
|
// it would have made. Either rule's match is noted as an offence, for the
|
||||||
// client's history. Its figures are the rule's id and action.
|
// client's history. Its figures are the rule's id and action.
|
||||||
type ruleFiles struct{}
|
type ruleFiles struct {
|
||||||
|
rules *rules.Files
|
||||||
|
metrics *metrics.Metrics
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (ruleFiles) name() string { return "rule_files" }
|
func (ruleFiles) name() string { return "rule_files" }
|
||||||
|
|
||||||
func (ruleFiles) check(
|
func (r ruleFiles) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
@@ -35,7 +40,7 @@ func (ruleFiles) check(
|
|||||||
|
|
||||||
for _, rule := range matched {
|
for _, rule := range matched {
|
||||||
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
|
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
|
||||||
rq.h.metrics.RuleMatched(rule.ID, rule.Action)
|
r.metrics.RuleMatched(rule.ID, rule.Action)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(matched) == 0 {
|
if len(matched) == 0 {
|
||||||
@@ -60,7 +65,8 @@ func (ruleFiles) check(
|
|||||||
rq.attack = true
|
rq.attack = true
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: rq.banForAttack(
|
action: r.banMaker.banForAttack(
|
||||||
|
rq,
|
||||||
now,
|
now,
|
||||||
bans.Notes{RuleID: last.ID, Target: last.Target},
|
bans.Notes{RuleID: last.ID, Target: last.Target},
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -12,13 +12,19 @@ import (
|
|||||||
|
|
||||||
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
|
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
|
||||||
// the first such tag, in sorted order.
|
// the first such tag, in sorted order.
|
||||||
type tagDeny struct{}
|
type tagDeny struct {
|
||||||
|
tags []string // SWW_WAF_TAG_DENY
|
||||||
|
}
|
||||||
|
|
||||||
func (tagDeny) name() string { return "tag_deny" }
|
func (tagDeny) name() string { return "tag_deny" }
|
||||||
|
|
||||||
func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
func (t tagDeny) check(
|
||||||
|
_ context.Context,
|
||||||
|
rq *request,
|
||||||
|
_ time.Time,
|
||||||
|
) *verdict {
|
||||||
for _, tag := range rq.line.Tags {
|
for _, tag := range rq.line.Tags {
|
||||||
if slices.Contains(rq.h.config.TagDeny, tag) {
|
if slices.Contains(t.tags, tag) {
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientDenied,
|
action: requestlog.ActionClientDenied,
|
||||||
figures: map[string]any{"tag": tag},
|
figures: map[string]any{"tag": tag},
|
||||||
|
|||||||
@@ -6,17 +6,25 @@ package waf
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that
|
// tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that
|
||||||
// holds the client, the tag being the list's name. It refuses nothing, and runs
|
// holds the client, the tag being the list's name. It refuses nothing, and runs
|
||||||
// first so that every refusal's log line carries these tags.
|
// first so that every refusal's log line carries these tags.
|
||||||
type tagLists struct{}
|
type tagLists struct {
|
||||||
|
lists *reputation.Lists
|
||||||
|
}
|
||||||
|
|
||||||
func (tagLists) name() string { return "tag_lists" }
|
func (tagLists) name() string { return "tag_lists" }
|
||||||
|
|
||||||
func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
func (t tagLists) check(
|
||||||
for _, name := range rq.h.lists.TagsOf(rq.client) {
|
_ context.Context,
|
||||||
|
rq *request,
|
||||||
|
_ time.Time,
|
||||||
|
) *verdict {
|
||||||
|
for _, name := range t.lists.TagsOf(rq.client) {
|
||||||
rq.addTag(name)
|
rq.addTag(name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,11 +16,14 @@ import (
|
|||||||
// it bans the client's netblock, or in observe mode raises the alert for the
|
// it bans the client's netblock, or in observe mode raises the alert for the
|
||||||
// ban it would have made. The ban's notes give the path as sent, and so does
|
// ban it would have made. The ban's notes give the path as sent, and so does
|
||||||
// its figure.
|
// its figure.
|
||||||
type trapPaths struct{}
|
type trapPaths struct {
|
||||||
|
paths []string // SWW_WAF_TRAP_PATHS
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (trapPaths) name() string { return "trap_paths" }
|
func (trapPaths) name() string { return "trap_paths" }
|
||||||
|
|
||||||
func (trapPaths) check(
|
func (t trapPaths) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
@@ -33,18 +36,22 @@ func (trapPaths) check(
|
|||||||
rq.attack = true
|
rq.attack = true
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: rq.banForAttack(now, bans.Notes{TrapPath: path}),
|
action: t.banMaker.banForAttack(
|
||||||
|
rq,
|
||||||
|
now,
|
||||||
|
bans.Notes{TrapPath: path},
|
||||||
|
),
|
||||||
figures: map[string]any{"path": path},
|
figures: map[string]any{"path": path},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// trapPath returns the request's path as sent, when its path as a path rule
|
// trapPath returns rq's path as sent, when its path as a path rule sees it,
|
||||||
// sees it, without the query, as sent or once percent-decoded, is in
|
// without the query, as sent or once percent-decoded, is in
|
||||||
// SWW_WAF_TRAP_PATHS, and "" otherwise.
|
// SWW_WAF_TRAP_PATHS, and "" otherwise.
|
||||||
func (rq *request) trapPath() string {
|
func (t trapPaths) trapPath(rq *request) string {
|
||||||
path := rules.Path(rq.in)
|
path := rules.Path(rq.in)
|
||||||
if !slices.Contains(rq.h.config.TrapPaths, path) &&
|
if !slices.Contains(t.paths, path) &&
|
||||||
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
|
!slices.Contains(t.paths, rules.DecodeOnce(path)) {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+11
-9
@@ -123,10 +123,10 @@ func New(params Params) *Server {
|
|||||||
GeoJS: h.geojs,
|
GeoJS: h.geojs,
|
||||||
Anomalies: h.anomalies,
|
Anomalies: h.anomalies,
|
||||||
LookupFile: h.lookupFile,
|
LookupFile: h.lookupFile,
|
||||||
Lists: h.lists,
|
Lists: params.Lists,
|
||||||
DNSBL: h.dnsbl,
|
DNSBL: params.DNSBL,
|
||||||
AbuseIPDB: h.abuseIPDB,
|
AbuseIPDB: params.AbuseIPDB,
|
||||||
RouteWatch: h.routeWatch,
|
RouteWatch: params.RouteWatch,
|
||||||
Metrics: h.metrics,
|
Metrics: h.metrics,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -154,8 +154,10 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
|
listedBy := rq.h.blocklists.lists.ListedBy(rq.client)
|
||||||
rq.overAWholeRateLimit(now)
|
|
||||||
|
return rq.h.blocklists.refusedByBlocklists(listedBy) ||
|
||||||
|
rq.h.rateLimits.overAWholeRateLimit(rq, now)
|
||||||
}
|
}
|
||||||
|
|
||||||
// answerQuestion runs the checks on the visitor's request a question
|
// answerQuestion runs the checks on the visitor's request a question
|
||||||
@@ -194,7 +196,7 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
|||||||
if refused == nil && !underOwnPrefix {
|
if refused == nil && !underOwnPrefix {
|
||||||
rq.allow()
|
rq.allow()
|
||||||
// In observe mode, what enforce mode would have refused counts here.
|
// In observe mode, what enforce mode would have refused counts here.
|
||||||
rq.runBanningHeuristics(r.Context(), afterAnswer())
|
rq.runBanningHeuristics(r.Context(), rq.h.afterAnswer)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -213,7 +215,7 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banAndCount(r.Context(), afterAnswer())
|
rq.banAndCount(r.Context(), rq.h.afterAnswer)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readQuestion returns the visitor's address and the body a question carries,
|
// readQuestion returns the visitor's address and the body a question carries,
|
||||||
@@ -486,7 +488,7 @@ func (h *handler) finishWith(
|
|||||||
if rq.allowed {
|
if rq.allowed {
|
||||||
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
||||||
|
|
||||||
rq.banAndCount(ctx, onReport())
|
rq.banAndCount(ctx, rq.h.onReport)
|
||||||
|
|
||||||
rq.h.metrics.RequestReported(
|
rq.h.metrics.RequestReported(
|
||||||
&rq.line,
|
&rq.line,
|
||||||
|
|||||||
Reference in new issue
Block a user