Compare commits

...
Author SHA1 Message Date
sneak e52edb9f62 requestlog: the package doc no longer says one line per request
check / check (push) Waiting to run
It said the request log is one JSON object per request, then listed
three for a request: two from waf and one from rproxy.

Model: opus-5-5
2026-11-19 18:43:32 +00:00
sneak 5667661dd8 SPEC: the request log is not one line per request
check / check (push) In progress
The package list said requestlog writes one JSON line per request. SPEC's
"The log" section gives a request waf answers three request log lines.

Model: opus-5-5
2026-11-19 18:23:52 +00:00
sneak 5ce35505bc Constructors take Params: ratelimit.New and processlog.New (closes #450)
ratelimit.New takes ratelimit.Params (Limits, MaxClients) in place of
two arguments by position. The process logger moves out of requestlog
into internal/processlog, whose New takes Params (Output, InstanceName,
Level); its two tests move with it unchanged. geojs.GeoJS keeps its
params whole, as DNSBL, AbuseIPDB and routewatch.Client do. SPEC and
README list the new package.

Model: opus-5-5
2026-11-19 18:23:52 +00:00
17 changed files with 327 additions and 245 deletions

No files matched your search

+5 -3
View File
@@ -906,9 +906,11 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
`internal/coreruleset`, `internal/geojs`, `internal/lookup`, `internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and `internal/reputation`, `internal/routewatch`, `internal/anomaly` and
`internal/alerts`: one check or source each. `internal/alerts`: one check or source each.
- `internal/state`, `internal/requestlog`, `internal/remotelog` and - `internal/state`: the state and cache files.
`internal/metrics`: the state and cache files, the log lines, the syslog - `internal/requestlog`: the request log.
sender and the metrics. - `internal/processlog`: the process's own messages.
- `internal/remotelog`: the syslog sender.
- `internal/metrics`: the metrics.
## Rationale ## Rationale
+4 -2
View File
@@ -104,8 +104,10 @@ never as a time.
its own. its own.
- `internal/state`: reads the state and cache files at start, takes in edits - `internal/state`: reads the state and cache files at start, takes in edits
while running, and writes the files when due and at the stop. while running, and writes the files when due and at the stop.
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and - `internal/requestlog`: the request log, as JSON lines on stdout.
sending them to the syslog server. - `internal/processlog`: the process lines on stdout, which hold the
process's own messages.
- `internal/remotelog`: sends the lines on stdout to the syslog server.
- `internal/metrics`: the metrics, in the Prometheus text format. - `internal/metrics`: the metrics, in the Prometheus text format.
- `Dockerfile`: the lint and test phases, then the image. - `Dockerfile`: the lint and test phases, then the image.
`deploy/example-app` is an app built on the image, with `rproxy` in its `deploy/example-app` is an app built on the image, with `rproxy` in its
+14 -24
View File
@@ -84,12 +84,7 @@ type Params struct {
// one request to GeoJS is under way at a time, and it asks about every // one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use. // client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct { type GeoJS struct {
url string params Params
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
// httpClient follows no redirect, so that visitors' addresses go to // httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure. // GeoJS alone: a redirect is a failure.
httpClient *http.Client httpClient *http.Client
@@ -145,12 +140,7 @@ func New(params Params) *GeoJS {
} }
return &GeoJS{ return &GeoJS{
url: params.URL, params: params,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
httpClient: &http.Client{ httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error { CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse return http.ErrUseLastResponse
@@ -193,7 +183,7 @@ func (g *GeoJS) LookUp(
answer, found := g.kept(client) answer, found := g.kept(client)
if !found { if !found {
g.wentWithoutAnswer() g.params.WentWithoutAnswer()
} }
w, waiting := g.waiting[client] w, waiting := g.waiting[client]
@@ -273,7 +263,7 @@ func (g *GeoJS) Load(answers []Answer) {
g.answers.Purge() g.answers.Purge()
now := g.now() now := g.params.Now()
for _, answer := range answers { for _, answer := range answers {
if now.Sub(answer.Answered) < keepFor { if now.Sub(answer.Answered) < keepFor {
@@ -310,7 +300,7 @@ func (g *GeoJS) answerOrWait(
} }
if w == nil { if w == nil {
g.wentWithoutAnswer() g.params.WentWithoutAnswer()
return Answer{}, nil // too many clients wait already return Answer{}, nil // too many clients wait already
} }
@@ -322,7 +312,7 @@ func (g *GeoJS) answerOrWait(
} }
if w.late { if w.late {
g.wentWithoutAnswer() g.params.WentWithoutAnswer()
return Answer{}, nil return Answer{}, nil
} }
@@ -333,7 +323,7 @@ func (g *GeoJS) answerOrWait(
// kept returns client's answer, if GeoJS gave it less than keepFor ago, // kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used. // and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) { func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.now() now := g.params.Now()
kept, found := g.answers.Get(client) kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor { if !found || now.Sub(kept.Answered) >= keepFor {
@@ -350,7 +340,7 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
// been called. The requests to GeoJS are for every client waiting, so they // been called. The requests to GeoJS are for every client waiting, so they
// go on when the caller whose ctx is given stops waiting, until StopAsking. // go on when the caller whose ctx is given stops waiting, until StopAsking.
func (g *GeoJS) ask(ctx context.Context) { func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.stopped || g.now().Before(g.retryAt) { if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
return return
} }
@@ -382,9 +372,9 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
kept, answered := g.keep(clients, given, err) kept, answered := g.keep(clients, given, err)
if g.answered != nil { if g.params.Answered != nil {
for _, answer := range kept { for _, answer := range kept {
g.answered(answer) g.params.Answered(answer)
} }
} }
@@ -433,7 +423,7 @@ func (g *GeoJS) keep(
g.mu.Lock() g.mu.Lock()
defer g.mu.Unlock() defer g.mu.Unlock()
now := g.now() now := g.params.Now()
kept := make([]Answer, 0, len(clients)) kept := make([]Answer, 0, len(clients))
leftOut := 0 leftOut := 0
@@ -477,7 +467,7 @@ func (g *GeoJS) keep(
w.late = true w.late = true
} }
g.failed(err, g.retryDelay) g.params.Failed(err, g.retryDelay)
return kept, false return kept, false
} }
@@ -507,7 +497,7 @@ func (g *GeoJS) request(
req, err := http.NewRequestWithContext( req, err := http.NewRequestWithContext(
ctx, ctx,
http.MethodGet, http.MethodGet,
g.url, g.params.URL,
http.NoBody, http.NoBody,
) )
if err != nil { if err != nil {
@@ -516,7 +506,7 @@ func (g *GeoJS) request(
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",") req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.asked() g.params.Asked()
res, err := g.httpClient.Do(req) res, err := g.httpClient.Do(req)
if err != nil { if err != nil {
+44
View File
@@ -0,0 +1,44 @@
// Package processlog builds the logger for the process's own messages,
// which it writes as JSON lines marked "type":"process". The request log
// is package requestlog's.
package processlog
import (
"io"
"log/slog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// Params are what New needs.
type Params struct {
// Output is where the lines are written.
Output io.Writer
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
InstanceName string
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
Level slog.Level
}
// New returns the logger for the process's own messages: JSON lines on
// params.Output, marked "type":"process", with the time in the same form
// as a request line's. It writes only the messages at params.Level or more
// severe; the request lines requestlog.Write writes are never held back.
func New(params Params) *slog.Logger {
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
Level: params.Level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
requestlog.FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", params.InstanceName)
}
+105
View File
@@ -0,0 +1,105 @@
package processlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/processlog"
)
// instanceName is the instance the tests' loggers are built with.
const instanceName = "host1/blog"
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: slog.LevelInfo,
}).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: level,
})
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+7 -4
View File
@@ -11,7 +11,7 @@ import (
func TestHistoryKeepsEveryRequest(t *testing.T) { func TestHistoryKeepsEveryRequest(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32") other := netip.MustParsePrefix("198.51.100.7/32")
start := midnight() start := midnight()
@@ -100,7 +100,10 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
func TestResetKeepsTheHistory(t *testing.T) { func TestResetKeepsTheHistory(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -123,7 +126,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) { func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
for client, requests := range map[string]int{ for client, requests := range map[string]int{
"198.51.100.9/32": 2, "198.51.100.9/32": 2,
+18 -6
View File
@@ -156,16 +156,28 @@ type Request struct {
BlockedByCoreRuleSet bool BlockedByCoreRuleSet bool
} }
// New returns a Limiter for limits, with no client counted yet, whose // Params are what New needs.
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past type Params struct {
// it, the least recently seen client is dropped, with its history, and // Limits are what each client's requests and bytes are counted against.
// starts afresh if it comes back. Limits Limits
func New(limits Limits, maxClients int) *Limiter { // MaxClients is the most clients the table holds,
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil) // SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
// is dropped, with its history, and starts afresh if it comes back.
MaxClients int
}
// New returns a Limiter with no client counted yet.
func New(params Params) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
params.MaxClients,
nil,
)
if err != nil { if err != nil {
panic(err) // NewLRU fails only for a size below one panic(err) // NewLRU fails only for a size below one
} }
limits := params.Limits
return &Limiter{ return &Limiter{
windows: [3]window{ windows: [3]window{
{ {
+63 -39
View File
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
now := midnight() now := midnight()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1}, Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
tableSize, MaxClients: tableSize,
) })
if limiter.OverARateLimit(client, now) { if limiter.OverARateLimit(client, now) {
t.Error("a client never seen is over a rate limit") t.Error("a client never seen is over a rate limit")
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
wantCount(t, limiter, client, now, minute) wantCount(t, limiter, client, now, minute)
// A limit that is off is never gone over. // A limit that is off is never gone over.
off := ratelimit.New(ratelimit.Limits{}, tableSize) off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
off.Count(client, now, whole) off.Count(client, now, whole)
if off.OverARateLimit(client, now) { if off.OverARateLimit(client, now) {
@@ -95,7 +95,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(tc.limits, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
quarter := tc.length / 4 quarter := tc.length / 4
@@ -126,10 +129,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerHour: limit}, Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -159,10 +162,13 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) { func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64}, Limits: ratelimit.Limits{
tableSize, PerMinute: 5,
) BytesPerDay: math.MaxInt64,
},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -203,10 +209,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
t.Parallel() t.Parallel()
// Only the hour has limits: the minute's and the day's are off. // Only the hour has limits: the minute's and the day's are off.
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}, Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -253,7 +259,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(tc.limits, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
// 600 bytes are within the limit, 600 more over it. // 600 bytes are within the limit, 600 more over it.
@@ -287,10 +296,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}, Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("203.0.113.10/32") other := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -314,7 +323,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
func TestCountGivesTheBytesInEachWindow(t *testing.T) { func TestCountGivesTheBytesInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -340,7 +349,10 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
func TestResetSetsTheBytesBackToZero(t *testing.T) { func TestResetSetsTheBytesBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{BytesPerDay: 1000},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -356,7 +368,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
) { ) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -411,7 +423,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
func TestCountGivesTheRequestsInEachWindow(t *testing.T) { func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -438,10 +450,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
func TestResetSetsTheCountsBackToZero(t *testing.T) { func TestResetSetsTheCountsBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerDay: limit}, Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -463,7 +475,10 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -482,10 +497,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
func TestRefusedRequestsCount(t *testing.T) { func TestRefusedRequestsCount(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}, Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
tableSize, MaxClients: tableSize,
) })
refused := netip.MustParsePrefix("203.0.113.9/32") refused := netip.MustParsePrefix("203.0.113.9/32")
within := netip.MustParsePrefix("203.0.113.10/32") within := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -518,7 +533,10 @@ func TestRefusedRequestsCount(t *testing.T) {
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -534,7 +552,10 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) { func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -562,7 +583,10 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
const maxClients = 3 const maxClients = 3
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: 1},
MaxClients: maxClients,
})
now := midnight() now := midnight()
clients := make([]netip.Prefix, maxClients+1) clients := make([]netip.Prefix, maxClients+1)
+12 -6
View File
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
"2001:db8::/64", "2001:db8::/64",
} }
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
for _, i := range []int{2, 3, 0, 1} { for _, i := range []int{2, 3, 0, 1} {
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole) limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
} }
@@ -52,7 +52,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) before := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
for range limit { for range limit {
wantCount(t, before, client, start, "") wantCount(t, before, client, start, "")
} }
@@ -60,7 +63,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
// Loaded into a new limiter, as across a restart, the client has no // Loaded into a new limiter, as across a restart, the client has no
// fresh allowance. // fresh allowance.
later := start.Add(time.Minute) later := start.Add(time.Minute)
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) after := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
after.Load(before.Snapshot(), later) after.Load(before.Snapshot(), later)
wantCount(t, after, client, later, hour) wantCount(t, after, client, later, hour)
} }
@@ -71,7 +77,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter.Count(client, start, whole) limiter.Count(client, start, whole)
limiter.CountBytes(client, start, 5, whole) limiter.CountBytes(client, start, 5, whole)
limiter.CountRefusal(client, start, limit) limiter.CountRefusal(client, start, limit)
@@ -80,7 +86,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
loaded := func(now time.Time) ratelimit.Client { loaded := func(now time.Time) ratelimit.Client {
t.Helper() t.Helper()
after := ratelimit.New(ratelimit.Limits{}, tableSize) after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
after.Load(limiter.Snapshot(), now) after.Load(limiter.Snapshot(), now)
return after.Snapshot()[0] return after.Snapshot()[0]
@@ -142,7 +148,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
addr = addr.Next() addr = addr.Next()
} }
limiter := ratelimit.New(ratelimit.Limits{}, maxClients) limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
limiter.Load(clients, midnight()) limiter.Load(clients, midnight())
got := limiter.Snapshot() got := limiter.Snapshot()
+4 -33
View File
@@ -1,7 +1,7 @@
// Package requestlog writes the lines sww prints on stdout: one // Package requestlog writes the request log sww prints on stdout: JSON
// JSON object per request, two from waf and a short one from rproxy, // lines marked "type":"request", at most two from waf and a short one
// marked "type":"request", and the process's own messages as JSON lines marked // from rproxy for each request. Package processlog writes the process's
// "type":"process". // own messages.
package requestlog package requestlog
import ( import (
@@ -9,7 +9,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"log/slog"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -300,31 +299,3 @@ func Milliseconds(d time.Duration) float64 {
time.Millisecond/time.Microsecond, time.Millisecond/time.Microsecond,
) )
} }
// NewProcessLogger returns the logger for the process's own messages:
// JSON lines on w, marked "type":"process", with the time in the same form
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
// the request lines Write writes are never held back.
func NewProcessLogger(
w io.Writer,
instanceName string,
level slog.Level,
) *slog.Logger {
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", instanceName)
}
-87
View File
@@ -3,8 +3,6 @@ package requestlog_test
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"log/slog"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -134,88 +132,3 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
t.Errorf("wrote %s, want %s", out.String(), want) t.Errorf("wrote %s, want %s", out.String(), want)
} }
} }
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := requestlog.NewProcessLogger(
&out,
"host1/blog",
level,
)
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+6 -6
View File
@@ -18,7 +18,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: processlog.New(processlog.Params{
io.Discard, Output: io.Discard,
"test", InstanceName: "test",
slog.LevelInfo, Level: slog.LevelInfo,
), }),
}) })
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
+6 -6
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -370,11 +370,11 @@ func startServerLogging(
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: requestLog, RequestLog: requestLog,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: processlog.New(processlog.Params{
processLog, Output: processLog,
"test", InstanceName: "test",
slog.LevelInfo, Level: slog.LevelInfo,
), }),
}) })
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
+1 -1
View File
@@ -2448,7 +2448,7 @@ func newParams(dir string) state.Params {
AttackBanDuration: 7 * 24 * time.Hour, AttackBanDuration: 7 * 24 * time.Hour,
MaxBans: 5000, MaxBans: 5000,
}), }),
Limiter: ratelimit.New(ratelimit.Limits{}, 20000), Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
GeoJS: geojs.New(geojs.Params{Now: midnight}), GeoJS: geojs.New(geojs.Params{Now: midnight}),
Lists: reputation.New(reputation.Params{ Lists: reputation.New(reputation.Params{
BlocklistURLs: []string{ BlocklistURLs: []string{
+6 -6
View File
@@ -10,7 +10,7 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
) )
@@ -23,11 +23,11 @@ import (
// setting for either. // setting for either.
func RunRProxy(ctx context.Context, params Params) int { func RunRProxy(ctx context.Context, params Params) int {
hostname, _ := os.Hostname() // "" when the host has no name to give hostname, _ := os.Hostname() // "" when the host has no name to give
processLog := requestlog.NewProcessLogger( processLog := processlog.New(processlog.Params{
params.Stdout, Output: params.Stdout,
hostname, InstanceName: hostname,
slog.LevelInfo, Level: slog.LevelInfo,
) })
cfg, err := config.RProxyFromEnvironment(params.Viper) cfg, err := config.RProxyFromEnvironment(params.Viper)
if err != nil { if err != nil {
+16 -10
View File
@@ -19,10 +19,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch" "sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state" "sneak.berlin/go/smallwebwaf/internal/state"
@@ -43,8 +43,11 @@ const remoteLogStopTimeout = 2 * time.Second
func RunWAF(ctx context.Context, params Params) int { func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's // Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through. // error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := requestlog.NewProcessLogger(params.Stdout, processLog := processlog.New(processlog.Params{
config.InstanceName(params.Viper), slog.LevelError) Output: params.Stdout,
InstanceName: config.InstanceName(params.Viper),
Level: slog.LevelError,
})
cfg, err := config.WAFFromEnvironment(params.Viper) cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil { if err != nil {
@@ -63,11 +66,11 @@ func RunWAF(ctx context.Context, params Params) int {
stdout = io.MultiWriter(params.Stdout, remote) stdout = io.MultiWriter(params.Stdout, remote)
} }
processLog = requestlog.NewProcessLogger( processLog = processlog.New(processlog.Params{
stdout, Output: stdout,
cfg.InstanceName, InstanceName: cfg.InstanceName,
cfg.LogLevel, Level: cfg.LogLevel,
) })
if remote != nil { if remote != nil {
stopSending := startSending(ctx, remote, processLog) stopSending := startSending(ctx, remote, processLog)
@@ -239,14 +242,17 @@ func newLookupFile(
// newLimiter returns the table of clients, with the rate and byte limits. // newLimiter returns the table of clients, with the rate and byte limits.
func newLimiter(cfg *config.WAF) *ratelimit.Limiter { func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
return ratelimit.New(ratelimit.Limits{ return ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute, PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour, PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay, PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute, BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour, BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay, BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients) },
MaxClients: cfg.MaxTrackedClients,
})
} }
// newLedger returns the ledger of the bans, with the settings for them. // newLedger returns the ledger of the bans, with the settings for them.
+16 -12
View File
@@ -24,6 +24,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -398,11 +399,11 @@ func startRProxy(
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: processlog.New(processlog.Params{
io.Discard, Output: io.Discard,
"rproxy", InstanceName: "rproxy",
slog.LevelInfo, Level: slog.LevelInfo,
), }),
}) })
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
@@ -471,11 +472,11 @@ func newWAFWith(
} }
out := &output{} out := &output{}
processLog := requestlog.NewProcessLogger( processLog := processlog.New(processlog.Params{
out, Output: out,
cfg.InstanceName, InstanceName: cfg.InstanceName,
cfg.LogLevel, Level: cfg.LogLevel,
) })
ruleFiles, err := rules.Load(rules.Params{ ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog, Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
@@ -561,14 +562,17 @@ func addCounters(params *waf.Params) {
cfg.MetricsBusiestCountriesAndASNs, cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName, cfg.InstanceName,
) )
params.Limiter = ratelimit.New(ratelimit.Limits{ params.Limiter = ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute, PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour, PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay, PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute, BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour, BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay, BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients) },
MaxClients: cfg.MaxTrackedClients,
})
params.Ledger = bans.New(bans.Params{ params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration, LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow, LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,