Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a0277879d7 | ||
|
|
2083584ee0 | ||
|
|
fe61d6b066 | ||
|
|
9a8ed745bd | ||
|
|
0a230c86ce | ||
|
|
40386dce6e |
No files matched your search
@@ -906,9 +906,13 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
- `internal/dirwatch`: watches a directory, so that `internal/rules` and
|
||||||
`internal/metrics`: the state and cache files, the log lines, the syslog
|
`internal/lookup` read their files again when they change.
|
||||||
sender and the metrics.
|
- `internal/state`: the state and cache files.
|
||||||
|
- `internal/requestlog`: the request log.
|
||||||
|
- `internal/processlog`: the process's own messages.
|
||||||
|
- `internal/remotelog`: the syslog sender.
|
||||||
|
- `internal/metrics`: the metrics.
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
|
|||||||
@@ -93,6 +93,9 @@ never as a time.
|
|||||||
imports no other package of this repository: `waf` gives it what it needs
|
imports no other package of this repository: `waf` gives it what it needs
|
||||||
to count and report failures.
|
to count and report failures.
|
||||||
- `internal/lookup`: looks clients up in the lookup database.
|
- `internal/lookup`: looks clients up in the lookup database.
|
||||||
|
- `internal/dirwatch`: watches a directory, and calls a function once it has
|
||||||
|
gone a short time without a change. `internal/rules` and `internal/lookup`
|
||||||
|
read their files again this way.
|
||||||
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
||||||
AbuseIPDB in the background, and keeps their answers. It also fetches
|
AbuseIPDB in the background, and keeps their answers. It also fetches
|
||||||
AbuseIPDB's blacklist, which `internal/waf` bans.
|
AbuseIPDB's blacklist, which `internal/waf` bans.
|
||||||
@@ -104,8 +107,10 @@ never as a time.
|
|||||||
its own.
|
its own.
|
||||||
- `internal/state`: reads the state and cache files at start, takes in edits
|
- `internal/state`: reads the state and cache files at start, takes in edits
|
||||||
while running, and writes the files when due and at the stop.
|
while running, and writes the files when due and at the stop.
|
||||||
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
|
- `internal/requestlog`: the request log, as JSON lines on stdout.
|
||||||
sending them to the syslog server.
|
- `internal/processlog`: the process lines on stdout, which hold the
|
||||||
|
process's own messages.
|
||||||
|
- `internal/remotelog`: sends the lines on stdout to the syslog server.
|
||||||
- `internal/metrics`: the metrics, in the Prometheus text format.
|
- `internal/metrics`: the metrics, in the Prometheus text format.
|
||||||
- `Dockerfile`: the lint and test phases, then the image.
|
- `Dockerfile`: the lint and test phases, then the image.
|
||||||
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
// Package dirwatch watches a directory, and calls a function once the
|
||||||
|
// directory has gone a short time without a change after one. The rule
|
||||||
|
// files and the lookup database are read again this way.
|
||||||
|
package dirwatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
)
|
||||||
|
|
||||||
|
// quietTime is how long the directory must go without a change before the
|
||||||
|
// function is called, so that a file still being written, such as one
|
||||||
|
// saved in place, appended to or copied in with scp, is read only once
|
||||||
|
// whole.
|
||||||
|
const quietTime = 2 * time.Second
|
||||||
|
|
||||||
|
// Watcher watches one directory. The directory is watched, rather than a
|
||||||
|
// file in it, so that a file replaced by a rename is seen.
|
||||||
|
type Watcher struct {
|
||||||
|
watcher *fsnotify.Watcher
|
||||||
|
}
|
||||||
|
|
||||||
|
// New starts watching dir, or returns why it cannot.
|
||||||
|
func New(dir string) (*Watcher, error) {
|
||||||
|
watcher, err := fsnotify.NewWatcher()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = watcher.Add(dir)
|
||||||
|
if err != nil {
|
||||||
|
_ = watcher.Close()
|
||||||
|
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Watcher{watcher: watcher}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close stops watching the directory.
|
||||||
|
func (w *Watcher) Close() error {
|
||||||
|
return w.watcher.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// CallWhenQuiet calls changed each time the directory has gone quietTime
|
||||||
|
// without a change, after a change to a path for which counts is true, until
|
||||||
|
// ctx is done. It also calls failed with each error the watch reports. The
|
||||||
|
// wait starts at once, as if for a change, so that a change made before the
|
||||||
|
// watch began is taken in too.
|
||||||
|
func (w *Watcher) CallWhenQuiet(
|
||||||
|
ctx context.Context,
|
||||||
|
counts func(path string) bool,
|
||||||
|
changed func(),
|
||||||
|
failed func(error),
|
||||||
|
) {
|
||||||
|
callWhenQuiet(
|
||||||
|
ctx, w.watcher.Events, w.watcher.Errors, counts, changed, failed,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// callWhenQuiet is CallWhenQuiet, for the changes from events and the
|
||||||
|
// errors from errs.
|
||||||
|
func callWhenQuiet(
|
||||||
|
ctx context.Context,
|
||||||
|
events <-chan fsnotify.Event,
|
||||||
|
errs <-chan error,
|
||||||
|
counts func(path string) bool,
|
||||||
|
changed func(),
|
||||||
|
failed func(error),
|
||||||
|
) {
|
||||||
|
quiet := time.NewTimer(quietTime)
|
||||||
|
defer quiet.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case event := <-events:
|
||||||
|
if counts(event.Name) {
|
||||||
|
quiet.Reset(quietTime)
|
||||||
|
}
|
||||||
|
case <-quiet.C:
|
||||||
|
changed()
|
||||||
|
case err := <-errs:
|
||||||
|
failed(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
package dirwatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
)
|
||||||
|
|
||||||
|
// errWatch is an error the watch reports.
|
||||||
|
var errWatch = errors.New("too many changes")
|
||||||
|
|
||||||
|
func TestDirectoryThatCannotBeWatchedIsAnError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, err := New(filepath.Join(t.TempDir(), "missing"))
|
||||||
|
if err == nil {
|
||||||
|
t.Error("a missing directory is watched, want an error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tests below run callWhenQuiet in a synctest bubble, where time is a
|
||||||
|
// clock of the test's own: time.Sleep moves it on at once, and
|
||||||
|
// synctest.Wait returns once callWhenQuiet waits again, so that every call
|
||||||
|
// due by then is made. The test sends the changes itself, as the watch of
|
||||||
|
// a directory cannot run in a bubble.
|
||||||
|
|
||||||
|
func TestSecondChangeStartsTheWaitAgain(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
loop := start(t, everyPath)
|
||||||
|
|
||||||
|
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 0)
|
||||||
|
|
||||||
|
// The second part starts the wait again.
|
||||||
|
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 0)
|
||||||
|
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 1)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCalledOnceQuietAfterTheStart(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
// An edit saved before the watch began sends no change, and is
|
||||||
|
// taken in by this call.
|
||||||
|
loop := start(t, everyPath)
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 1)
|
||||||
|
|
||||||
|
// Long after, it has been called only once.
|
||||||
|
time.Sleep(time.Hour)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 1)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChangeThatDoesNotCountIsNoChange(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
loop := start(t, func(path string) bool {
|
||||||
|
return path == "ipinfo_lite.mmdb"
|
||||||
|
})
|
||||||
|
|
||||||
|
// The wait that starts with the watch ends with a call.
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
loop.changes <- fsnotify.Event{Name: "other.mmdb", Op: fsnotify.Create}
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 1)
|
||||||
|
|
||||||
|
loop.changes <- fsnotify.Event{Name: "ipinfo_lite.mmdb", Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 2)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestErrorsOfTheWatchHandedOn(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
loop := start(t, everyPath)
|
||||||
|
|
||||||
|
loop.errs <- errWatch
|
||||||
|
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
if !errors.Is(loop.failure, errWatch) {
|
||||||
|
t.Errorf("failed with %v, want %v", loop.failure, errWatch)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// loop is a callWhenQuiet that a test runs.
|
||||||
|
type loop struct {
|
||||||
|
// changes and errs send it changes and errors.
|
||||||
|
changes chan fsnotify.Event
|
||||||
|
errs chan error
|
||||||
|
// calls counts its calls of changed, and failure is the last error it
|
||||||
|
// handed to failed.
|
||||||
|
calls atomic.Int32
|
||||||
|
failure error
|
||||||
|
}
|
||||||
|
|
||||||
|
// start runs callWhenQuiet, with counts, until the test ends.
|
||||||
|
func start(t *testing.T, counts func(path string) bool) *loop {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
l := &loop{changes: make(chan fsnotify.Event), errs: make(chan error)}
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
callWhenQuiet(ctx, l.changes, l.errs, counts,
|
||||||
|
func() { l.calls.Add(1) },
|
||||||
|
func(err error) { l.failure = err },
|
||||||
|
)
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
stop()
|
||||||
|
<-stopped
|
||||||
|
})
|
||||||
|
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
|
||||||
|
// everyPath counts a change to any path.
|
||||||
|
func everyPath(string) bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantCalls checks how many times changed has been called.
|
||||||
|
func (l *loop) wantCalls(t *testing.T, want int32) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
got := l.calls.Load()
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("changed called %d times, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
+14
-24
@@ -84,12 +84,7 @@ type Params struct {
|
|||||||
// one request to GeoJS is under way at a time, and it asks about every
|
// one request to GeoJS is under way at a time, and it asks about every
|
||||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||||
type GeoJS struct {
|
type GeoJS struct {
|
||||||
url string
|
params Params
|
||||||
answered func(Answer)
|
|
||||||
now func() time.Time
|
|
||||||
asked func()
|
|
||||||
failed func(err error, askingAgainIn time.Duration)
|
|
||||||
wentWithoutAnswer func()
|
|
||||||
// httpClient follows no redirect, so that visitors' addresses go to
|
// httpClient follows no redirect, so that visitors' addresses go to
|
||||||
// GeoJS alone: a redirect is a failure.
|
// GeoJS alone: a redirect is a failure.
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
@@ -145,12 +140,7 @@ func New(params Params) *GeoJS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &GeoJS{
|
return &GeoJS{
|
||||||
url: params.URL,
|
params: params,
|
||||||
answered: params.Answered,
|
|
||||||
now: params.Now,
|
|
||||||
asked: params.Asked,
|
|
||||||
failed: params.Failed,
|
|
||||||
wentWithoutAnswer: params.WentWithoutAnswer,
|
|
||||||
httpClient: &http.Client{
|
httpClient: &http.Client{
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
@@ -193,7 +183,7 @@ func (g *GeoJS) LookUp(
|
|||||||
|
|
||||||
answer, found := g.kept(client)
|
answer, found := g.kept(client)
|
||||||
if !found {
|
if !found {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
}
|
}
|
||||||
|
|
||||||
w, waiting := g.waiting[client]
|
w, waiting := g.waiting[client]
|
||||||
@@ -273,7 +263,7 @@ func (g *GeoJS) Load(answers []Answer) {
|
|||||||
|
|
||||||
g.answers.Purge()
|
g.answers.Purge()
|
||||||
|
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
|
|
||||||
for _, answer := range answers {
|
for _, answer := range answers {
|
||||||
if now.Sub(answer.Answered) < keepFor {
|
if now.Sub(answer.Answered) < keepFor {
|
||||||
@@ -310,7 +300,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w == nil {
|
if w == nil {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil // too many clients wait already
|
return Answer{}, nil // too many clients wait already
|
||||||
}
|
}
|
||||||
@@ -322,7 +312,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w.late {
|
if w.late {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil
|
return Answer{}, nil
|
||||||
}
|
}
|
||||||
@@ -333,7 +323,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
||||||
// and notes that it was used.
|
// and notes that it was used.
|
||||||
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
|
|
||||||
kept, found := g.answers.Get(client)
|
kept, found := g.answers.Get(client)
|
||||||
if !found || now.Sub(kept.Answered) >= keepFor {
|
if !found || now.Sub(kept.Answered) >= keepFor {
|
||||||
@@ -350,7 +340,7 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
// been called. The requests to GeoJS are for every client waiting, so they
|
// been called. The requests to GeoJS are for every client waiting, so they
|
||||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.stopped || g.now().Before(g.retryAt) {
|
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -382,9 +372,9 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.answered != nil {
|
if g.params.Answered != nil {
|
||||||
for _, answer := range kept {
|
for _, answer := range kept {
|
||||||
g.answered(answer)
|
g.params.Answered(answer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -433,7 +423,7 @@ func (g *GeoJS) keep(
|
|||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
kept := make([]Answer, 0, len(clients))
|
kept := make([]Answer, 0, len(clients))
|
||||||
leftOut := 0
|
leftOut := 0
|
||||||
|
|
||||||
@@ -477,7 +467,7 @@ func (g *GeoJS) keep(
|
|||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
g.failed(err, g.retryDelay)
|
g.params.Failed(err, g.retryDelay)
|
||||||
|
|
||||||
return kept, false
|
return kept, false
|
||||||
}
|
}
|
||||||
@@ -507,7 +497,7 @@ func (g *GeoJS) request(
|
|||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
g.url,
|
g.params.URL,
|
||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -516,7 +506,7 @@ func (g *GeoJS) request(
|
|||||||
|
|
||||||
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
||||||
|
|
||||||
g.asked()
|
g.params.Asked()
|
||||||
|
|
||||||
res, err := g.httpClient.Do(req)
|
res, err := g.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+18
-49
@@ -13,18 +13,13 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
"github.com/oschwald/maxminddb-golang/v2"
|
"github.com/oschwald/maxminddb-golang/v2"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
)
|
)
|
||||||
|
|
||||||
// quietTime is how long the lookup database must go without a change
|
|
||||||
// before it is read again, so that a file still being copied in is read
|
|
||||||
// only once whole.
|
|
||||||
const quietTime = 2 * time.Second
|
|
||||||
|
|
||||||
// FileParams are what OpenFile needs.
|
// FileParams are what OpenFile needs.
|
||||||
type FileParams struct {
|
type FileParams struct {
|
||||||
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
||||||
@@ -122,20 +117,13 @@ func (f *File) ReadFailures() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Watch watches the directory of the lookup database until ctx is done,
|
// Watch watches the directory of the lookup database until ctx is done,
|
||||||
// and reads the file again once it has gone without a change for
|
// and reads the file again once it has gone without a change for a short
|
||||||
// quietTime, after it is replaced, written or removed, and after Watch
|
// time, after it is replaced, written or removed, and after Watch starts
|
||||||
// starts watching. If the directory cannot be watched, that is logged, and
|
// watching, as dirwatch describes. A change to another file in its
|
||||||
// the database read at start stays in use.
|
// directory does not count. If the directory cannot be watched, that is
|
||||||
|
// logged, and the database read at start stays in use.
|
||||||
func (f *File) Watch(ctx context.Context) {
|
func (f *File) Watch(ctx context.Context) {
|
||||||
watcher, err := fsnotify.NewWatcher()
|
watcher, err := dirwatch.New(filepath.Dir(f.params.Path))
|
||||||
if err == nil {
|
|
||||||
defer func() {
|
|
||||||
_ = watcher.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
err = watcher.Add(filepath.Dir(f.params.Path))
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.params.ProcessLog.Error(
|
f.params.ProcessLog.Error(
|
||||||
"cannot watch the lookup database for replacements",
|
"cannot watch the lookup database for replacements",
|
||||||
@@ -146,41 +134,22 @@ func (f *File) Watch(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = watcher.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
||||||
"file", f.params.Path)
|
"file", f.params.Path)
|
||||||
|
|
||||||
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
watcher.CallWhenQuiet(ctx, f.isDatabase, f.readAgain, func(err error) {
|
||||||
|
f.params.ProcessLog.Warn("watching the lookup database failed",
|
||||||
|
"error", err.Error())
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAfterChanges reads the lookup database again once quietTime has
|
// isDatabase reports whether path names the lookup database.
|
||||||
// passed without a change to it from events, until ctx is done, and logs
|
func (f *File) isDatabase(path string) bool {
|
||||||
// the errors from errs. A change to another file in its directory does not
|
return filepath.Clean(path) == filepath.Clean(f.params.Path)
|
||||||
// count. The wait starts at once, as if for a change, so that a file
|
|
||||||
// replaced after OpenFile read it, and before its directory was watched,
|
|
||||||
// is read too.
|
|
||||||
func (f *File) readAfterChanges(
|
|
||||||
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
|
||||||
) {
|
|
||||||
path := filepath.Clean(f.params.Path)
|
|
||||||
|
|
||||||
quiet := time.NewTimer(quietTime)
|
|
||||||
defer quiet.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case event := <-events:
|
|
||||||
if filepath.Clean(event.Name) == path {
|
|
||||||
quiet.Reset(quietTime)
|
|
||||||
}
|
|
||||||
case <-quiet.C:
|
|
||||||
f.readAgain()
|
|
||||||
case err := <-errs:
|
|
||||||
f.params.ProcessLog.Warn("watching the lookup database failed",
|
|
||||||
"error", err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAgain reads the lookup database again, in place of the one in use,
|
// readAgain reads the lookup database again, in place of the one in use,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package lookup
|
package lookup
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -12,7 +11,6 @@ import (
|
|||||||
"testing/synctest"
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
"github.com/maxmind/mmdbwriter/mmdbtype"
|
"github.com/maxmind/mmdbwriter/mmdbtype"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
@@ -133,66 +131,48 @@ func TestFileThatCannotBeReadIsAnError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The tests below run readAfterChanges in a synctest bubble, where time is
|
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
||||||
// a clock of the test's own: time.Sleep moves it on at once, and
|
t.Parallel()
|
||||||
// synctest.Wait returns once readAfterChanges waits again, so that every
|
|
||||||
// reading due by then is done. The test sends the changes itself, as the
|
|
||||||
// watch of a directory cannot run in a bubble.
|
|
||||||
|
|
||||||
func TestReplacementCopiedOverTheFileInTwoPartsIsReadOnlyWhole(
|
dir := t.TempDir()
|
||||||
t *testing.T,
|
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||||
) {
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
|
f := openFile(t, path, newQueue())
|
||||||
|
|
||||||
|
for changed, want := range map[string]bool{
|
||||||
|
path: true,
|
||||||
|
dir + "/./ipinfo_lite.mmdb": true,
|
||||||
|
filepath.Join(dir, "other.mmdb"): false,
|
||||||
|
} {
|
||||||
|
if f.isDatabase(changed) != want {
|
||||||
|
t.Errorf(
|
||||||
|
"a change to %s counts: %t, want %t",
|
||||||
|
changed,
|
||||||
|
!want,
|
||||||
|
want,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tests below run in a synctest bubble, where time is a clock of the
|
||||||
|
// test's own, so that the time a file is read can be checked. The wait for
|
||||||
|
// the directory to go quiet before a reading is tested in dirwatch.
|
||||||
|
|
||||||
|
func TestReplacementIsReadAgain(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
|
||||||
writeDatabase(t, path, "DE")
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
queue := newQueue()
|
queue := newQueue()
|
||||||
f := openFile(t, path, queue)
|
f := openFile(t, path, queue)
|
||||||
changes := watch(t, f)
|
|
||||||
|
|
||||||
other := filepath.Join(dir, "replacement.mmdb")
|
writeDatabase(t, path, "KP")
|
||||||
writeDatabase(t, other, "KP")
|
time.Sleep(time.Hour)
|
||||||
|
f.readAgain()
|
||||||
//nolint:gosec // a file the test wrote
|
|
||||||
replacement, err := os.ReadFile(other)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read %s: %v", other, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The file in use is overwritten in place, and keeps giving what
|
|
||||||
// it gave. Its first part alone is not a .mmdb file.
|
|
||||||
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("create %s: %v", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = file.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
half := len(replacement) / 2
|
|
||||||
write(t, file, replacement[:half])
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "DE")
|
|
||||||
|
|
||||||
// The second part starts the wait again.
|
|
||||||
write(t, file, replacement[half:])
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "DE")
|
|
||||||
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "KP")
|
wantCountry(t, f, "KP")
|
||||||
|
|
||||||
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
||||||
@@ -219,15 +199,10 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
queue := newQueue()
|
queue := newQueue()
|
||||||
f := openFile(t, path, queue)
|
f := openFile(t, path, queue)
|
||||||
read := f.LastRead()
|
read := f.LastRead()
|
||||||
changes := watch(t, f)
|
|
||||||
|
|
||||||
writeFile(t, path, "not a lookup database\n")
|
writeFile(t, path, "not a lookup database\n")
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
// Long after, the replacement has been read once.
|
|
||||||
time.Sleep(time.Hour)
|
time.Sleep(time.Hour)
|
||||||
synctest.Wait()
|
f.readAgain()
|
||||||
wantCountry(t, f, "DE")
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
||||||
@@ -240,7 +215,7 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
}
|
}
|
||||||
|
|
||||||
wantAlerts(t, queue, alerts.Alert{
|
wantAlerts(t, queue, alerts.Alert{
|
||||||
Time: read.Add(quietTime),
|
Time: read.Add(time.Hour),
|
||||||
Event: alerts.EventFileError,
|
Event: alerts.EventFileError,
|
||||||
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
||||||
Detail: map[string]any{
|
Detail: map[string]any{
|
||||||
@@ -252,55 +227,6 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
|
||||||
writeDatabase(t, path, "DE")
|
|
||||||
|
|
||||||
f := openFile(t, path, newQueue())
|
|
||||||
changes := watch(t, f)
|
|
||||||
|
|
||||||
// The wait that starts with the watch ends with a reading.
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
writeDatabase(t, path, "KP")
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: filepath.Join(dir, "other.mmdb"), Op: fsnotify.Create}
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "DE")
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "KP")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReplacementSavedBeforeTheWatchStartsIsRead(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
|
||||||
writeDatabase(t, path, "DE")
|
|
||||||
|
|
||||||
f := openFile(t, path, newQueue())
|
|
||||||
|
|
||||||
// Saved after OpenFile read the file, and before its directory was
|
|
||||||
// watched, so that no change is seen for it.
|
|
||||||
writeDatabase(t, path, "KP")
|
|
||||||
watch(t, f)
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "KP")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newQueue returns a queue of alerts for a webhook that is never sent
|
// newQueue returns a queue of alerts for a webhook that is never sent
|
||||||
// them, so that they wait in it for the test to look at.
|
// them, so that they wait in it for the test to look at.
|
||||||
func newQueue() *alerts.Queue {
|
func newQueue() *alerts.Queue {
|
||||||
@@ -344,28 +270,6 @@ func openFile(t *testing.T, path string, queue *alerts.Queue) *File {
|
|||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
// watch runs f's readAfterChanges until the test ends, and returns the
|
|
||||||
// channel that sends it changes.
|
|
||||||
func watch(t *testing.T, f *File) chan<- fsnotify.Event {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
changes := make(chan fsnotify.Event)
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
f.readAfterChanges(ctx, changes, nil)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
stop()
|
|
||||||
<-stopped
|
|
||||||
})
|
|
||||||
|
|
||||||
return changes
|
|
||||||
}
|
|
||||||
|
|
||||||
// wantCountry checks the country f gives testClient.
|
// wantCountry checks the country f gives testClient.
|
||||||
func wantCountry(t *testing.T, f *File, want string) {
|
func wantCountry(t *testing.T, f *File, want string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -401,13 +305,3 @@ func writeFile(t *testing.T, path, content string) {
|
|||||||
t.Fatalf("write %s: %v", path, err)
|
t.Fatalf("write %s: %v", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// write writes data to the end of file.
|
|
||||||
func write(t *testing.T, file *os.File, data []byte) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := file.Write(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
// Package processlog builds the logger for the process's own messages,
|
||||||
|
// which it writes as JSON lines marked "type":"process". The request log
|
||||||
|
// is package requestlog's.
|
||||||
|
package processlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Params are what New needs.
|
||||||
|
type Params struct {
|
||||||
|
// Output is where the lines are written.
|
||||||
|
Output io.Writer
|
||||||
|
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
|
||||||
|
InstanceName string
|
||||||
|
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
|
||||||
|
Level slog.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns the logger for the process's own messages: JSON lines on
|
||||||
|
// params.Output, marked "type":"process", with the time in the same form
|
||||||
|
// as a request line's. It writes only the messages at params.Level or more
|
||||||
|
// severe; the request lines requestlog.Write writes are never held back.
|
||||||
|
func New(params Params) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
|
||||||
|
Level: params.Level,
|
||||||
|
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
||||||
|
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
||||||
|
return slog.String(
|
||||||
|
slog.TimeKey,
|
||||||
|
requestlog.FormatTime(attr.Value.Time()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return attr
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return slog.New(handler).
|
||||||
|
With("type", "process", "instance", params.InstanceName)
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package processlog_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// instanceName is the instance the tests' loggers are built with.
|
||||||
|
const instanceName = "host1/blog"
|
||||||
|
|
||||||
|
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processlog.New(processlog.Params{
|
||||||
|
Output: &out,
|
||||||
|
InstanceName: instanceName,
|
||||||
|
Level: slog.LevelInfo,
|
||||||
|
}).
|
||||||
|
Info("starting",
|
||||||
|
"version", "v1")
|
||||||
|
|
||||||
|
var fields map[string]any
|
||||||
|
|
||||||
|
err := json.Unmarshal(out.Bytes(), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", out.String(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields["type"] != "process" ||
|
||||||
|
fields["instance"] != "host1/blog" ||
|
||||||
|
fields["msg"] != "starting" ||
|
||||||
|
fields["level"] != "INFO" ||
|
||||||
|
fields["version"] != "v1" {
|
||||||
|
t.Errorf("process line %v", fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeText, _ := fields["time"].(string)
|
||||||
|
|
||||||
|
logged, err := time.Parse(time.RFC3339, timeText)
|
||||||
|
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
||||||
|
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
||||||
|
time.Since(logged) > time.Minute {
|
||||||
|
t.Errorf(
|
||||||
|
"process line time %q, want now in UTC with milliseconds",
|
||||||
|
timeText,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
levels := []slog.Level{
|
||||||
|
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, level := range levels {
|
||||||
|
t.Run(level.String(), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processLog := processlog.New(processlog.Params{
|
||||||
|
Output: &out,
|
||||||
|
InstanceName: instanceName,
|
||||||
|
Level: level,
|
||||||
|
})
|
||||||
|
for _, at := range levels {
|
||||||
|
processLog.Log(t.Context(), at, "message")
|
||||||
|
}
|
||||||
|
|
||||||
|
var got, want []string
|
||||||
|
|
||||||
|
for line := range strings.Lines(out.String()) {
|
||||||
|
var fields struct {
|
||||||
|
Level string `json:"level"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(line), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", line, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got = append(got, fields.Level)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, written := range levels[i:] {
|
||||||
|
want = append(want, written.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("lines at %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
|||||||
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -100,7 +100,10 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
|||||||
func TestResetKeepsTheHistory(t *testing.T) {
|
func TestResetKeepsTheHistory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -123,7 +126,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
|
|||||||
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
|
|
||||||
for client, requests := range map[string]int{
|
for client, requests := range map[string]int{
|
||||||
"198.51.100.9/32": 2,
|
"198.51.100.9/32": 2,
|
||||||
|
|||||||
@@ -156,16 +156,28 @@ type Request struct {
|
|||||||
BlockedByCoreRuleSet bool
|
BlockedByCoreRuleSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns a Limiter for limits, with no client counted yet, whose
|
// Params are what New needs.
|
||||||
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
|
type Params struct {
|
||||||
// it, the least recently seen client is dropped, with its history, and
|
// Limits are what each client's requests and bytes are counted against.
|
||||||
// starts afresh if it comes back.
|
Limits Limits
|
||||||
func New(limits Limits, maxClients int) *Limiter {
|
// MaxClients is the most clients the table holds,
|
||||||
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
|
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
|
||||||
|
// is dropped, with its history, and starts afresh if it comes back.
|
||||||
|
MaxClients int
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns a Limiter with no client counted yet.
|
||||||
|
func New(params Params) *Limiter {
|
||||||
|
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
|
||||||
|
params.MaxClients,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // NewLRU fails only for a size below one
|
panic(err) // NewLRU fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
|
limits := params.Limits
|
||||||
|
|
||||||
return &Limiter{
|
return &Limiter{
|
||||||
windows: [3]window{
|
windows: [3]window{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
if limiter.OverARateLimit(client, now) {
|
||||||
t.Error("a client never seen is over a rate limit")
|
t.Error("a client never seen is over a rate limit")
|
||||||
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
wantCount(t, limiter, client, now, minute)
|
wantCount(t, limiter, client, now, minute)
|
||||||
|
|
||||||
// A limit that is off is never gone over.
|
// A limit that is off is never gone over.
|
||||||
off := ratelimit.New(ratelimit.Limits{}, tableSize)
|
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
off.Count(client, now, whole)
|
off.Count(client, now, whole)
|
||||||
|
|
||||||
if off.OverARateLimit(client, now) {
|
if off.OverARateLimit(client, now) {
|
||||||
@@ -95,7 +95,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(tc.limits, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: tc.limits,
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
quarter := tc.length / 4
|
quarter := tc.length / 4
|
||||||
@@ -126,10 +129,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -159,10 +162,13 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
|||||||
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
|
Limits: ratelimit.Limits{
|
||||||
tableSize,
|
PerMinute: 5,
|
||||||
)
|
BytesPerDay: math.MaxInt64,
|
||||||
|
},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -203,10 +209,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Only the hour has limits: the minute's and the day's are off.
|
// Only the hour has limits: the minute's and the day's are off.
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -253,7 +259,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(tc.limits, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: tc.limits,
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// 600 bytes are within the limit, 600 more over it.
|
// 600 bytes are within the limit, 600 more over it.
|
||||||
@@ -287,10 +296,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("203.0.113.10/32")
|
other := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -314,7 +323,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
|||||||
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -340,7 +349,10 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{BytesPerDay: 1000},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -356,7 +368,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -411,7 +423,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -438,10 +450,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -463,7 +475,10 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
|||||||
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -482,10 +497,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
|||||||
func TestRefusedRequestsCount(t *testing.T) {
|
func TestRefusedRequestsCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
refused := netip.MustParsePrefix("203.0.113.9/32")
|
refused := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
within := netip.MustParsePrefix("203.0.113.10/32")
|
within := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -518,7 +533,10 @@ func TestRefusedRequestsCount(t *testing.T) {
|
|||||||
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -534,7 +552,10 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
|||||||
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -562,7 +583,10 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
|
|||||||
|
|
||||||
const maxClients = 3
|
const maxClients = 3
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: 1},
|
||||||
|
MaxClients: maxClients,
|
||||||
|
})
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
clients := make([]netip.Prefix, maxClients+1)
|
clients := make([]netip.Prefix, maxClients+1)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
|
|||||||
"2001:db8::/64",
|
"2001:db8::/64",
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
for _, i := range []int{2, 3, 0, 1} {
|
for _, i := range []int{2, 3, 0, 1} {
|
||||||
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
||||||
}
|
}
|
||||||
@@ -52,7 +52,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
before := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
for range limit {
|
for range limit {
|
||||||
wantCount(t, before, client, start, "")
|
wantCount(t, before, client, start, "")
|
||||||
}
|
}
|
||||||
@@ -60,7 +63,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
// Loaded into a new limiter, as across a restart, the client has no
|
// Loaded into a new limiter, as across a restart, the client has no
|
||||||
// fresh allowance.
|
// fresh allowance.
|
||||||
later := start.Add(time.Minute)
|
later := start.Add(time.Minute)
|
||||||
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
after := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
after.Load(before.Snapshot(), later)
|
after.Load(before.Snapshot(), later)
|
||||||
wantCount(t, after, client, later, hour)
|
wantCount(t, after, client, later, hour)
|
||||||
}
|
}
|
||||||
@@ -71,7 +77,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
limiter.Count(client, start, whole)
|
limiter.Count(client, start, whole)
|
||||||
limiter.CountBytes(client, start, 5, whole)
|
limiter.CountBytes(client, start, 5, whole)
|
||||||
limiter.CountRefusal(client, start, limit)
|
limiter.CountRefusal(client, start, limit)
|
||||||
@@ -80,7 +86,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
loaded := func(now time.Time) ratelimit.Client {
|
loaded := func(now time.Time) ratelimit.Client {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
after := ratelimit.New(ratelimit.Limits{}, tableSize)
|
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
after.Load(limiter.Snapshot(), now)
|
after.Load(limiter.Snapshot(), now)
|
||||||
|
|
||||||
return after.Snapshot()[0]
|
return after.Snapshot()[0]
|
||||||
@@ -142,7 +148,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
|
|||||||
addr = addr.Next()
|
addr = addr.Next()
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
|
||||||
limiter.Load(clients, midnight())
|
limiter.Load(clients, midnight())
|
||||||
|
|
||||||
got := limiter.Snapshot()
|
got := limiter.Snapshot()
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Package requestlog writes the lines sww prints on stdout: one
|
// Package requestlog writes the request log sww prints on stdout: JSON
|
||||||
// JSON object per request, two from waf and a short one from rproxy,
|
// lines marked "type":"request", at most two from waf and a short one
|
||||||
// marked "type":"request", and the process's own messages as JSON lines marked
|
// from rproxy for each request. Package processlog writes the process's
|
||||||
// "type":"process".
|
// own messages.
|
||||||
package requestlog
|
package requestlog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
@@ -300,31 +299,3 @@ func Milliseconds(d time.Duration) float64 {
|
|||||||
time.Millisecond/time.Microsecond,
|
time.Millisecond/time.Microsecond,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewProcessLogger returns the logger for the process's own messages:
|
|
||||||
// JSON lines on w, marked "type":"process", with the time in the same form
|
|
||||||
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
|
|
||||||
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
|
|
||||||
// the request lines Write writes are never held back.
|
|
||||||
func NewProcessLogger(
|
|
||||||
w io.Writer,
|
|
||||||
instanceName string,
|
|
||||||
level slog.Level,
|
|
||||||
) *slog.Logger {
|
|
||||||
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
|
|
||||||
Level: level,
|
|
||||||
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
|
||||||
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
|
||||||
return slog.String(
|
|
||||||
slog.TimeKey,
|
|
||||||
FormatTime(attr.Value.Time()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return attr
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
return slog.New(handler).
|
|
||||||
With("type", "process", "instance", instanceName)
|
|
||||||
}
|
|
||||||
@@ -3,8 +3,6 @@ package requestlog_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"log/slog"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -134,88 +132,3 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
|
|||||||
t.Errorf("wrote %s, want %s", out.String(), want)
|
t.Errorf("wrote %s, want %s", out.String(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
|
|
||||||
Info("starting",
|
|
||||||
"version", "v1")
|
|
||||||
|
|
||||||
var fields map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(out.Bytes(), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", out.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fields["type"] != "process" ||
|
|
||||||
fields["instance"] != "host1/blog" ||
|
|
||||||
fields["msg"] != "starting" ||
|
|
||||||
fields["level"] != "INFO" ||
|
|
||||||
fields["version"] != "v1" {
|
|
||||||
t.Errorf("process line %v", fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
timeText, _ := fields["time"].(string)
|
|
||||||
|
|
||||||
logged, err := time.Parse(time.RFC3339, timeText)
|
|
||||||
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
|
||||||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
|
||||||
time.Since(logged) > time.Minute {
|
|
||||||
t.Errorf(
|
|
||||||
"process line time %q, want now in UTC with milliseconds",
|
|
||||||
timeText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
levels := []slog.Level{
|
|
||||||
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, level := range levels {
|
|
||||||
t.Run(level.String(), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processLog := requestlog.NewProcessLogger(
|
|
||||||
&out,
|
|
||||||
"host1/blog",
|
|
||||||
level,
|
|
||||||
)
|
|
||||||
for _, at := range levels {
|
|
||||||
processLog.Log(t.Context(), at, "message")
|
|
||||||
}
|
|
||||||
|
|
||||||
var got, want []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(out.String()) {
|
|
||||||
var fields struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", line, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = append(got, fields.Level)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, written := range levels[i:] {
|
|
||||||
want = append(want, written.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("lines at %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
io.Discard,
|
Output: io.Discard,
|
||||||
"test",
|
InstanceName: "test",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
||||||
@@ -870,10 +870,14 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
|
|||||||
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
// rproxy is waiting for the rest of the body well within this limit, even on
|
||||||
|
// a busy host, and answers when it runs out, well before dial's deadline.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "3000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1262,7 +1266,9 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const limit = 2 * time.Second
|
// The app's answer arrives well within this limit, even on a busy host, and
|
||||||
|
// the connection closes when it runs out, well before dial's deadline.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, answerBeforeTheBody)
|
app := startApp(t, answerBeforeTheBody)
|
||||||
@@ -1427,17 +1433,20 @@ func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(
|
|||||||
|
|
||||||
const answer = "the end of a long answer"
|
const answer = "the end of a long answer"
|
||||||
|
|
||||||
|
// The request reaches the app well within this limit, even on a busy host.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
_ = http.NewResponseController(w).Flush()
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
time.Sleep(2100 * time.Millisecond)
|
time.Sleep(limit + 100*time.Millisecond)
|
||||||
|
|
||||||
_, _ = io.WriteString(w, answer)
|
_, _ = io.WriteString(w, answer)
|
||||||
})
|
})
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "2000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
got := visit(
|
got := visit(
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -383,11 +383,11 @@ func newServer(
|
|||||||
return rproxy.New(rproxy.Params{
|
return rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
processLog,
|
Output: processLog,
|
||||||
"test",
|
InstanceName: "test",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1452,14 +1452,18 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
|
|||||||
|
|
||||||
greet(w, r)
|
greet(w, r)
|
||||||
})
|
})
|
||||||
addr, out := startRProxy(
|
out := &output{}
|
||||||
t,
|
server := newServer(
|
||||||
standIn.URL,
|
t, standIn.URL, app.URL,
|
||||||
app.URL,
|
map[string]string{wafBackendTimeout: "200"},
|
||||||
map[string]string{
|
out, out,
|
||||||
wafBackendTimeout: "200",
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
// The breaker's clock stands still, as in
|
||||||
|
// TestAnswersThatCannotBePassedOnOpenTheBreaker: on a slow host, the five
|
||||||
|
// failures could take more than five seconds, and the next request could
|
||||||
|
// come more than a second later.
|
||||||
|
server.StopBreakerClock(time.Now())
|
||||||
|
addr := serve(t, server)
|
||||||
|
|
||||||
// An ask can time out before it reaches the stand-in, so these
|
// An ask can time out before it reaches the stand-in, so these
|
||||||
// may leave fewer than five questions, or leave some late.
|
// may leave fewer than five questions, or leave some late.
|
||||||
@@ -1831,11 +1835,15 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
// The app's answer to the upgrade arrives well within these limits, even on a
|
||||||
|
// busy host.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, echoAfterUpgrade)
|
app := startApp(t, echoAfterUpgrade)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "2000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
maxResponseTime: "2000",
|
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1860,7 +1868,10 @@ func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Past both time limits, the connection still carries what is sent.
|
// Past both time limits, the connection still carries what is sent.
|
||||||
time.Sleep(2100 * time.Millisecond)
|
time.Sleep(limit + 100*time.Millisecond)
|
||||||
|
|
||||||
|
// dial's deadline ran from before the upgrade and this wait.
|
||||||
|
_ = conn.SetDeadline(time.Now().Add(waitLimit))
|
||||||
|
|
||||||
const message = "still here\n"
|
const message = "still here\n"
|
||||||
|
|
||||||
|
|||||||
+19
-48
@@ -18,12 +18,11 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
"golang.org/x/net/http/httpguts"
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The actions a rule takes when it matches.
|
// The actions a rule takes when it matches.
|
||||||
@@ -40,12 +39,6 @@ const (
|
|||||||
// extension ends the name of every rule file.
|
// extension ends the name of every rule file.
|
||||||
const extension = ".rules"
|
const extension = ".rules"
|
||||||
|
|
||||||
// quietTime is how long SWW_WAF_RULE_FILES_DIR must go without a change before
|
|
||||||
// the rule files are read again, so that a file still being written, such
|
|
||||||
// as one saved in place, appended to or copied in with scp, is read only
|
|
||||||
// once whole.
|
|
||||||
const quietTime = 2 * time.Second
|
|
||||||
|
|
||||||
// headerTarget starts the target that is one request header,
|
// headerTarget starts the target that is one request header,
|
||||||
// header:<Name>.
|
// header:<Name>.
|
||||||
const headerTarget = "header:"
|
const headerTarget = "header:"
|
||||||
@@ -176,26 +169,18 @@ func (f *Files) Len() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Watch watches Dir until ctx is done, and reads the rule files again
|
// Watch watches Dir until ctx is done, and reads the rule files again
|
||||||
// once Dir has had no change for quietTime, after one is edited, added or
|
// once Dir has had no change for a short time, after one is edited, added
|
||||||
// removed, and after Watch starts watching. If they then hold an error,
|
// or removed, and after Watch starts watching, as dirwatch describes. If
|
||||||
// the rules stay as they were, the error is logged with its file and
|
// they then hold an error, the rules stay as they were, the error is
|
||||||
// line, and the files are read again after the next change. If Dir cannot
|
// logged with its file and line, and the files are read again after the
|
||||||
// be watched, that is logged, and the rules stay as they were loaded.
|
// next change. If Dir cannot be watched, that is logged, and the rules
|
||||||
// While Enabled is false, Watch returns at once.
|
// stay as they were loaded. While Enabled is false, Watch returns at once.
|
||||||
func (f *Files) Watch(ctx context.Context) {
|
func (f *Files) Watch(ctx context.Context) {
|
||||||
if !f.params.Enabled {
|
if !f.params.Enabled {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
watcher, err := fsnotify.NewWatcher()
|
watcher, err := dirwatch.New(f.params.Dir)
|
||||||
if err == nil {
|
|
||||||
defer func() {
|
|
||||||
_ = watcher.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
err = watcher.Add(f.params.Dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
||||||
"error", err.Error())
|
"error", err.Error())
|
||||||
@@ -203,36 +188,22 @@ func (f *Files) Watch(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = watcher.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
f.params.ProcessLog.Info("watching the rule files for edits",
|
f.params.ProcessLog.Info("watching the rule files for edits",
|
||||||
"directory", f.params.Dir)
|
"directory", f.params.Dir)
|
||||||
|
|
||||||
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
// A change to any file in Dir counts.
|
||||||
}
|
watcher.CallWhenQuiet(ctx,
|
||||||
|
func(string) bool { return true },
|
||||||
// readAfterChanges reads the rule files again once quietTime has passed
|
f.readAgain,
|
||||||
// without a change from events, until ctx is done, and logs the errors
|
func(err error) {
|
||||||
// from errs. The wait starts at once, as if for a change, so that an edit
|
|
||||||
// saved after Load read the files, and before Dir was watched, is taken
|
|
||||||
// in too.
|
|
||||||
func (f *Files) readAfterChanges(
|
|
||||||
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
|
||||||
) {
|
|
||||||
quiet := time.NewTimer(quietTime)
|
|
||||||
defer quiet.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-events:
|
|
||||||
quiet.Reset(quietTime)
|
|
||||||
case <-quiet.C:
|
|
||||||
f.readAgain()
|
|
||||||
case err := <-errs:
|
|
||||||
f.params.ProcessLog.Warn("watching the rule files failed",
|
f.params.ProcessLog.Warn("watching the rule files failed",
|
||||||
"error", err.Error())
|
"error", err.Error())
|
||||||
}
|
},
|
||||||
}
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAgain reads the rule files again, in place of the rules loaded, or
|
// readAgain reads the rule files again, in place of the rules loaded, or
|
||||||
|
|||||||
@@ -1,165 +0,0 @@
|
|||||||
package rules
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The tests below run readAfterChanges in a synctest bubble, where time is
|
|
||||||
// a clock of the test's own: time.Sleep moves it on at once, and
|
|
||||||
// synctest.Wait returns once readAfterChanges waits again, so that every
|
|
||||||
// reading due by then is done. The test sends the changes itself, as the
|
|
||||||
// watch of a directory cannot run in a bubble.
|
|
||||||
|
|
||||||
func TestFileWrittenInTwoPartsTakenInOnlyWhole(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "50-app.rules")
|
|
||||||
writeFile(t, path, "first path block ^/first\n")
|
|
||||||
files := load(t, dir)
|
|
||||||
changes := run(t, files)
|
|
||||||
|
|
||||||
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("create: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = file.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
// The first part ends in the middle of a ban rule's regex, which,
|
|
||||||
// read then, would ban every request.
|
|
||||||
write(t, file, "first path block ^/first\nprobe path ban ^/")
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantMatched(t, files, "/anything")
|
|
||||||
|
|
||||||
// The second part starts the wait again.
|
|
||||||
write(t, file, `\.env$`+"\n")
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantMatched(t, files, "/.env")
|
|
||||||
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantMatched(t, files, "/.env", "probe")
|
|
||||||
wantMatched(t, files, "/anything")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEditSavedBeforeTheWatchStartsTakenIn(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "50-app.rules")
|
|
||||||
writeFile(t, path, "first path block ^/first\n")
|
|
||||||
files := load(t, dir)
|
|
||||||
|
|
||||||
// Saved after Load read the files, and before the directory was
|
|
||||||
// watched, so that no change is seen for it.
|
|
||||||
writeFile(t, path, "first path block ^/edited\n")
|
|
||||||
run(t, files)
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
wantMatched(t, files, "/edited", "first")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// load loads the rules in dir.
|
|
||||||
func load(t *testing.T, dir string) *Files {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
files, err := Load(Params{
|
|
||||||
Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler),
|
|
||||||
Alerts: alerts.New(alerts.Params{}),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("load: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return files
|
|
||||||
}
|
|
||||||
|
|
||||||
// run runs files' readAfterChanges until the test ends, and returns the
|
|
||||||
// channel that sends it changes.
|
|
||||||
func run(t *testing.T, files *Files) chan<- fsnotify.Event {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
changes := make(chan fsnotify.Event)
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
files.readAfterChanges(ctx, changes, nil)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
stop()
|
|
||||||
<-stopped
|
|
||||||
})
|
|
||||||
|
|
||||||
return changes
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeFile writes content to the file at path.
|
|
||||||
func writeFile(t *testing.T, path, content string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
err := os.WriteFile(path, []byte(content), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write %s: %v", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// write writes text to the end of file.
|
|
||||||
func write(t *testing.T, file *os.File, text string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := file.WriteString(text)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// wantMatched checks the ids of the rules that a GET request for path
|
|
||||||
// matches, in order.
|
|
||||||
func wantMatched(t *testing.T, files *Files, path string, want ...string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
r := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
|
||||||
"http://app.example"+path, nil)
|
|
||||||
|
|
||||||
matched := files.Match(r)
|
|
||||||
|
|
||||||
got := make([]string, 0, len(matched))
|
|
||||||
for _, rule := range matched {
|
|
||||||
got = append(got, rule.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("%s matched %v, want %v", path, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2452,7 +2452,7 @@ func newParams(dir string) state.Params {
|
|||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}),
|
}),
|
||||||
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
|
||||||
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
||||||
Lists: reputation.New(reputation.Params{
|
Lists: reputation.New(reputation.Params{
|
||||||
BlocklistURLs: []string{
|
BlocklistURLs: []string{
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,11 +23,11 @@ import (
|
|||||||
// setting for either.
|
// setting for either.
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
func RunRProxy(ctx context.Context, params Params) int {
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
hostname, _ := os.Hostname() // "" when the host has no name to give
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := processlog.New(processlog.Params{
|
||||||
params.Stdout,
|
Output: params.Stdout,
|
||||||
hostname,
|
InstanceName: hostname,
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
)
|
})
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+22
-16
@@ -19,10 +19,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
@@ -43,8 +43,11 @@ const remoteLogStopTimeout = 2 * time.Second
|
|||||||
func RunWAF(ctx context.Context, params Params) int {
|
func RunWAF(ctx context.Context, params Params) int {
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
||||||
processLog := requestlog.NewProcessLogger(params.Stdout,
|
processLog := processlog.New(processlog.Params{
|
||||||
config.InstanceName(params.Viper), slog.LevelError)
|
Output: params.Stdout,
|
||||||
|
InstanceName: config.InstanceName(params.Viper),
|
||||||
|
Level: slog.LevelError,
|
||||||
|
})
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(params.Viper)
|
cfg, err := config.WAFFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -63,11 +66,11 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
stdout = io.MultiWriter(params.Stdout, remote)
|
stdout = io.MultiWriter(params.Stdout, remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
processLog = requestlog.NewProcessLogger(
|
processLog = processlog.New(processlog.Params{
|
||||||
stdout,
|
Output: stdout,
|
||||||
cfg.InstanceName,
|
InstanceName: cfg.InstanceName,
|
||||||
cfg.LogLevel,
|
Level: cfg.LogLevel,
|
||||||
)
|
})
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
stopSending := startSending(ctx, remote, processLog)
|
stopSending := startSending(ctx, remote, processLog)
|
||||||
@@ -239,14 +242,17 @@ func newLookupFile(
|
|||||||
|
|
||||||
// newLimiter returns the table of clients, with the rate and byte limits.
|
// newLimiter returns the table of clients, with the rate and byte limits.
|
||||||
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
||||||
return ratelimit.New(ratelimit.Limits{
|
return ratelimit.New(ratelimit.Params{
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
Limits: ratelimit.Limits{
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
}, cfg.MaxTrackedClients)
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
|
},
|
||||||
|
MaxClients: cfg.MaxTrackedClients,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLedger returns the ledger of the bans, with the settings for them.
|
// newLedger returns the ledger of the bans, with the settings for them.
|
||||||
|
|||||||
@@ -548,6 +548,10 @@ func (s *sender) requestWithHeader(
|
|||||||
// body is sent as it is after the headers, header holding its
|
// body is sent as it is after the headers, header holding its
|
||||||
// Content-Length or Transfer-Encoding. header may hold several lines,
|
// Content-Length or Transfer-Encoding. header may hold several lines,
|
||||||
// separated by "\r\n". It returns the whole answer.
|
// separated by "\r\n". It returns the whole answer.
|
||||||
|
//
|
||||||
|
// It waits for the answer as long as it takes, since a busy host can slow
|
||||||
|
// the Core Rule Set past any shorter wait. go test's -timeout ends a test
|
||||||
|
// whose answer never comes.
|
||||||
func (s *sender) requestWithBody(
|
func (s *sender) requestWithBody(
|
||||||
method, from, path, header, body string, status int, action string,
|
method, from, path, header, body string, status int, action string,
|
||||||
) (logLine, answer) {
|
) (logLine, answer) {
|
||||||
@@ -566,11 +570,6 @@ func (s *sender) requestWithBody(
|
|||||||
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
|
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
|
||||||
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
|
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
|
||||||
|
|
||||||
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
|
|
||||||
if err != nil {
|
|
||||||
s.t.Fatalf("set read deadline: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var got answer
|
var got answer
|
||||||
|
|
||||||
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
||||||
|
|||||||
+24
-24
@@ -24,6 +24,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
@@ -193,11 +194,11 @@ func (o *output) requestLine(t *testing.T) logLine {
|
|||||||
// endedLines waits for count requests that went through rproxy, or to waf's
|
// endedLines waits for count requests that went through rproxy, or to waf's
|
||||||
// own endpoints, to have ended, and returns the last log line of each, in the
|
// own endpoints, to have ended, and returns the last log line of each, in the
|
||||||
// order waf took them. A question's last line is the one its report writes.
|
// order waf took them. A question's last line is the one its report writes.
|
||||||
|
// It waits as long as it takes, as requestWithBody does.
|
||||||
func (o *output) endedLines(t *testing.T, count int) []logLine {
|
func (o *output) endedLines(t *testing.T, count int) []logLine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
deadline := time.Now().Add(waitLimit)
|
for {
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
found := o.lastLines(t)
|
found := o.lastLines(t)
|
||||||
|
|
||||||
ended := len(found) >= count
|
ended := len(found) >= count
|
||||||
@@ -213,10 +214,6 @@ func (o *output) endedLines(t *testing.T, count int) []logLine {
|
|||||||
|
|
||||||
time.Sleep(pollInterval)
|
time.Sleep(pollInterval)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// lastLines returns the last request log line written so far of each request,
|
// lastLines returns the last request log line written so far of each request,
|
||||||
@@ -398,11 +395,11 @@ func startRProxy(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
io.Discard,
|
Output: io.Discard,
|
||||||
"rproxy",
|
InstanceName: "rproxy",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
@@ -471,11 +468,11 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
|
|
||||||
out := &output{}
|
out := &output{}
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := processlog.New(processlog.Params{
|
||||||
out,
|
Output: out,
|
||||||
cfg.InstanceName,
|
InstanceName: cfg.InstanceName,
|
||||||
cfg.LogLevel,
|
Level: cfg.LogLevel,
|
||||||
)
|
})
|
||||||
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
||||||
@@ -561,14 +558,17 @@ func addCounters(params *waf.Params) {
|
|||||||
cfg.MetricsBusiestCountriesAndASNs,
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
)
|
)
|
||||||
params.Limiter = ratelimit.New(ratelimit.Limits{
|
params.Limiter = ratelimit.New(ratelimit.Params{
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
Limits: ratelimit.Limits{
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
}, cfg.MaxTrackedClients)
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
|
},
|
||||||
|
MaxClients: cfg.MaxTrackedClients,
|
||||||
|
})
|
||||||
params.Ledger = bans.New(bans.Params{
|
params.Ledger = bans.New(bans.Params{
|
||||||
LimitBanDuration: cfg.LimitBanDuration,
|
LimitBanDuration: cfg.LimitBanDuration,
|
||||||
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
||||||
|
|||||||
Reference in new issue
Block a user