From 1002433c3593f6e2a517f063417a3eea93fa601c Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 17:38:09 +0000 Subject: [PATCH] Constructors take Params: ratelimit.New and processlog.New (closes #450) ratelimit.New takes ratelimit.Params (Limits, MaxClients) in place of two arguments by position. The process logger moves out of requestlog into internal/processlog, whose New takes Params (Output, InstanceName, Level); its two tests move with it unchanged. geojs.GeoJS keeps its params whole, as DNSBL, AbuseIPDB and routewatch.Client do. SPEC and README list the new package. Model: opus-5-5 --- README.md | 8 +- SPEC.md | 7 +- internal/geojs/geojs.go | 38 ++++----- internal/processlog/processlog.go | 44 +++++++++++ internal/processlog/processlog_test.go | 105 +++++++++++++++++++++++++ internal/ratelimit/history_test.go | 11 ++- internal/ratelimit/ratelimit.go | 24 ++++-- internal/ratelimit/ratelimit_test.go | 102 +++++++++++++++--------- internal/ratelimit/snapshot_test.go | 18 +++-- internal/requestlog/requestlog.go | 35 +-------- internal/requestlog/requestlog_test.go | 87 -------------------- internal/rproxy/limits_test.go | 12 +-- internal/rproxy/rproxy_test.go | 12 +-- internal/state/state_test.go | 2 +- internal/sww/rproxy.go | 12 +-- internal/sww/waf.go | 38 +++++---- internal/waf/proxy_test.go | 40 +++++----- 17 files changed, 339 insertions(+), 256 deletions(-) create mode 100644 internal/processlog/processlog.go create mode 100644 internal/processlog/processlog_test.go diff --git a/README.md b/README.md index 9ab4147..56fb88b 100644 --- a/README.md +++ b/README.md @@ -906,9 +906,11 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)). `internal/coreruleset`, `internal/geojs`, `internal/lookup`, `internal/reputation`, `internal/routewatch`, `internal/anomaly` and `internal/alerts`: one check or source each. -- `internal/state`, `internal/requestlog`, `internal/remotelog` and - `internal/metrics`: the state and cache files, the log lines, the syslog - sender and the metrics. +- `internal/state`: the state and cache files. +- `internal/requestlog`: the request log. +- `internal/processlog`: the process's own messages. +- `internal/remotelog`: the syslog sender. +- `internal/metrics`: the metrics. ## Rationale diff --git a/SPEC.md b/SPEC.md index 5849f66..d38eb57 100644 --- a/SPEC.md +++ b/SPEC.md @@ -104,8 +104,11 @@ never as a time. its own. - `internal/state`: reads the state and cache files at start, takes in edits while running, and writes the files when due and at the stop. - - `internal/requestlog` and `internal/remotelog`: the lines on stdout, and - sending them to the syslog server. + - `internal/requestlog`: the request log, one JSON line per request on + stdout. + - `internal/processlog`: the process lines on stdout, which hold the + process's own messages. + - `internal/remotelog`: sends the lines on stdout to the syslog server. - `internal/metrics`: the metrics, in the Prometheus text format. - `Dockerfile`: the lint and test phases, then the image. `deploy/example-app` is an app built on the image, with `rproxy` in its diff --git a/internal/geojs/geojs.go b/internal/geojs/geojs.go index 962bc51..5141604 100644 --- a/internal/geojs/geojs.go +++ b/internal/geojs/geojs.go @@ -84,12 +84,7 @@ type Params struct { // one request to GeoJS is under way at a time, and it asks about every // client waiting, up to maxPerRequest. It is safe for concurrent use. type GeoJS struct { - url string - answered func(Answer) - now func() time.Time - asked func() - failed func(err error, askingAgainIn time.Duration) - wentWithoutAnswer func() + params Params // httpClient follows no redirect, so that visitors' addresses go to // GeoJS alone: a redirect is a failure. httpClient *http.Client @@ -140,12 +135,7 @@ func New(params Params) *GeoJS { } return &GeoJS{ - url: params.URL, - answered: params.Answered, - now: params.Now, - asked: params.Asked, - failed: params.Failed, - wentWithoutAnswer: params.WentWithoutAnswer, + params: params, httpClient: &http.Client{ CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse @@ -188,7 +178,7 @@ func (g *GeoJS) LookUp( answer, found := g.kept(client) if !found { - g.wentWithoutAnswer() + g.params.WentWithoutAnswer() } w, waiting := g.waiting[client] @@ -251,7 +241,7 @@ func (g *GeoJS) Load(answers []Answer) { g.answers.Purge() - now := g.now() + now := g.params.Now() for _, answer := range answers { if now.Sub(answer.Answered) < keepFor { @@ -288,7 +278,7 @@ func (g *GeoJS) answerOrWait( } if w == nil { - g.wentWithoutAnswer() + g.params.WentWithoutAnswer() return Answer{}, nil // too many clients wait already } @@ -299,7 +289,7 @@ func (g *GeoJS) answerOrWait( } if w.late { - g.wentWithoutAnswer() + g.params.WentWithoutAnswer() return Answer{}, nil } @@ -310,7 +300,7 @@ func (g *GeoJS) answerOrWait( // kept returns client's answer, if GeoJS gave it less than keepFor ago, // and notes that it was used. func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) { - now := g.now() + now := g.params.Now() kept, found := g.answers.Get(client) if !found || now.Sub(kept.Answered) >= keepFor { @@ -327,7 +317,7 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) { // GeoJS are for every client waiting, so they go on when the caller whose // ctx is given stops waiting. func (g *GeoJS) ask(ctx context.Context) { - if g.asking || g.now().Before(g.retryAt) { + if g.asking || g.params.Now().Before(g.retryAt) { return } @@ -349,9 +339,9 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) { given, err := g.request(ctx, clients) kept, answered := g.keep(clients, given, err) - if g.answered != nil { + if g.params.Answered != nil { for _, answer := range kept { - g.answered(answer) + g.params.Answered(answer) } } @@ -400,7 +390,7 @@ func (g *GeoJS) keep( g.mu.Lock() defer g.mu.Unlock() - now := g.now() + now := g.params.Now() kept := make([]Answer, 0, len(clients)) leftOut := 0 @@ -444,7 +434,7 @@ func (g *GeoJS) keep( w.late = true } - g.failed(err, g.retryDelay) + g.params.Failed(err, g.retryDelay) return kept, false } @@ -474,7 +464,7 @@ func (g *GeoJS) request( req, err := http.NewRequestWithContext( ctx, http.MethodGet, - g.url, + g.params.URL, http.NoBody, ) if err != nil { @@ -483,7 +473,7 @@ func (g *GeoJS) request( req.URL.RawQuery = "ip=" + strings.Join(addrs, ",") - g.asked() + g.params.Asked() res, err := g.httpClient.Do(req) if err != nil { diff --git a/internal/processlog/processlog.go b/internal/processlog/processlog.go new file mode 100644 index 0000000..e91e4f0 --- /dev/null +++ b/internal/processlog/processlog.go @@ -0,0 +1,44 @@ +// Package processlog builds the logger for the process's own messages, +// which it writes as JSON lines marked "type":"process". The request log +// is package requestlog's. +package processlog + +import ( + "io" + "log/slog" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// Params are what New needs. +type Params struct { + // Output is where the lines are written. + Output io.Writer + // InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance. + InstanceName string + // Level, SWW_WAF_LOG_LEVEL, is the least severe level written. + Level slog.Level +} + +// New returns the logger for the process's own messages: JSON lines on +// params.Output, marked "type":"process", with the time in the same form +// as a request line's. It writes only the messages at params.Level or more +// severe; the request lines requestlog.Write writes are never held back. +func New(params Params) *slog.Logger { + handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{ + Level: params.Level, + ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr { + if attr.Key == slog.TimeKey && len(groups) == 0 { + return slog.String( + slog.TimeKey, + requestlog.FormatTime(attr.Value.Time()), + ) + } + + return attr + }, + }) + + return slog.New(handler). + With("type", "process", "instance", params.InstanceName) +} diff --git a/internal/processlog/processlog_test.go b/internal/processlog/processlog_test.go new file mode 100644 index 0000000..277da4a --- /dev/null +++ b/internal/processlog/processlog_test.go @@ -0,0 +1,105 @@ +package processlog_test + +import ( + "bytes" + "encoding/json" + "log/slog" + "slices" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/processlog" +) + +// instanceName is the instance the tests' loggers are built with. +const instanceName = "host1/blog" + +func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + processlog.New(processlog.Params{ + Output: &out, + InstanceName: instanceName, + Level: slog.LevelInfo, + }). + Info("starting", + "version", "v1") + + var fields map[string]any + + err := json.Unmarshal(out.Bytes(), &fields) + if err != nil { + t.Fatalf("decode %q: %v", out.String(), err) + } + + if fields["type"] != "process" || + fields["instance"] != "host1/blog" || + fields["msg"] != "starting" || + fields["level"] != "INFO" || + fields["version"] != "v1" { + t.Errorf("process line %v", fields) + } + + timeText, _ := fields["time"].(string) + + logged, err := time.Parse(time.RFC3339, timeText) + if err != nil || !strings.HasSuffix(timeText, "Z") || + len(timeText) != len("2006-01-02T15:04:05.000Z") || + time.Since(logged) > time.Minute { + t.Errorf( + "process line time %q, want now in UTC with milliseconds", + timeText, + ) + } +} + +func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) { + t.Parallel() + + levels := []slog.Level{ + slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError, + } + + for i, level := range levels { + t.Run(level.String(), func(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + processLog := processlog.New(processlog.Params{ + Output: &out, + InstanceName: instanceName, + Level: level, + }) + for _, at := range levels { + processLog.Log(t.Context(), at, "message") + } + + var got, want []string + + for line := range strings.Lines(out.String()) { + var fields struct { + Level string `json:"level"` + } + + err := json.Unmarshal([]byte(line), &fields) + if err != nil { + t.Fatalf("decode %q: %v", line, err) + } + + got = append(got, fields.Level) + } + + for _, written := range levels[i:] { + want = append(want, written.String()) + } + + if !slices.Equal(got, want) { + t.Errorf("lines at %v, want %v", got, want) + } + }) + } +} diff --git a/internal/ratelimit/history_test.go b/internal/ratelimit/history_test.go index 74be757..af361ee 100644 --- a/internal/ratelimit/history_test.go +++ b/internal/ratelimit/history_test.go @@ -11,7 +11,7 @@ import ( func TestHistoryKeepsEveryRequest(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}, tableSize) + limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) { func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}, tableSize) + limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) client := netip.MustParsePrefix("203.0.113.9/32") other := netip.MustParsePrefix("198.51.100.7/32") start := midnight() @@ -100,7 +100,10 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { func TestResetKeepsTheHistory(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerMinute: limit}, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -123,7 +126,7 @@ func TestResetKeepsTheHistory(t *testing.T) { func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}, tableSize) + limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) for client, requests := range map[string]int{ "198.51.100.9/32": 2, diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 5e7ac0d..876efe5 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -156,16 +156,28 @@ type Request struct { BlockedByCoreRuleSet bool } -// New returns a Limiter for limits, with no client counted yet, whose -// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past -// it, the least recently seen client is dropped, with its history, and -// starts afresh if it comes back. -func New(limits Limits, maxClients int) *Limiter { - clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil) +// Params are what New needs. +type Params struct { + // Limits are what each client's requests and bytes are counted against. + Limits Limits + // MaxClients is the most clients the table holds, + // SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client + // is dropped, with its history, and starts afresh if it comes back. + MaxClients int +} + +// New returns a Limiter with no client counted yet. +func New(params Params) *Limiter { + clients, err := simplelru.NewLRU[netip.Prefix, *Client]( + params.MaxClients, + nil, + ) if err != nil { panic(err) // NewLRU fails only for a size below one } + limits := params.Limits + return &Limiter{ windows: [3]window{ { diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go index ec3ab1c..b11d289 100644 --- a/internal/ratelimit/ratelimit_test.go +++ b/internal/ratelimit/ratelimit_test.go @@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing( client := netip.MustParsePrefix("203.0.113.9/32") now := midnight() - limiter := ratelimit.New( - ratelimit.Limits{PerMinute: limit, PerDay: limit + 1}, - tableSize, - ) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1}, + MaxClients: tableSize, + }) if limiter.OverARateLimit(client, now) { t.Error("a client never seen is over a rate limit") @@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing( wantCount(t, limiter, client, now, minute) // A limit that is off is never gone over. - off := ratelimit.New(ratelimit.Limits{}, tableSize) + off := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) off.Count(client, now, whole) if off.OverARateLimit(client, now) { @@ -95,7 +95,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) { t.Run(tc.window, func(t *testing.T) { t.Parallel() - limiter := ratelimit.New(tc.limits, tableSize) + limiter := ratelimit.New(ratelimit.Params{ + Limits: tc.limits, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() quarter := tc.length / 4 @@ -126,10 +129,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) { func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { t.Parallel() - limiter := ratelimit.New( - ratelimit.Limits{PerMinute: limit, PerHour: limit}, - tableSize, - ) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit}, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -159,10 +162,13 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) { t.Parallel() - limiter := ratelimit.New( - ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64}, - tableSize, - ) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{ + PerMinute: 5, + BytesPerDay: math.MaxInt64, + }, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -203,10 +209,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) { t.Parallel() // Only the hour has limits: the minute's and the day's are off. - limiter := ratelimit.New( - ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}, - tableSize, - ) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -253,7 +259,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) { t.Run(tc.window, func(t *testing.T) { t.Parallel() - limiter := ratelimit.New(tc.limits, tableSize) + limiter := ratelimit.New(ratelimit.Params{ + Limits: tc.limits, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") // 600 bytes are within the limit, 600 more over it. @@ -287,10 +296,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) { func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { t.Parallel() - limiter := ratelimit.New( - ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}, - tableSize, - ) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") other := netip.MustParsePrefix("203.0.113.10/32") start := midnight() @@ -314,7 +323,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { func TestCountGivesTheBytesInEachWindow(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}, tableSize) + limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -340,7 +349,10 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) { func TestResetSetsTheBytesBackToZero(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{BytesPerDay: 1000}, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -356,7 +368,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit( ) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}, tableSize) + limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -411,7 +423,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit( func TestCountGivesTheRequestsInEachWindow(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}, tableSize) + limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -438,10 +450,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) { func TestResetSetsTheCountsBackToZero(t *testing.T) { t.Parallel() - limiter := ratelimit.New( - ratelimit.Limits{PerMinute: limit, PerDay: limit}, - tableSize, - ) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit}, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -463,7 +475,10 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) { func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerHour: limit}, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -482,10 +497,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { func TestRefusedRequestsCount(t *testing.T) { t.Parallel() - limiter := ratelimit.New( - ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}, - tableSize, - ) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}, + MaxClients: tableSize, + }) refused := netip.MustParsePrefix("203.0.113.9/32") within := netip.MustParsePrefix("203.0.113.10/32") start := midnight() @@ -518,7 +533,10 @@ func TestRefusedRequestsCount(t *testing.T) { func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerMinute: limit}, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -534,7 +552,10 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerHour: limit}, + MaxClients: tableSize, + }) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -562,7 +583,10 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) { const maxClients = 3 - limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients) + limiter := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerMinute: 1}, + MaxClients: maxClients, + }) now := midnight() clients := make([]netip.Prefix, maxClients+1) diff --git a/internal/ratelimit/snapshot_test.go b/internal/ratelimit/snapshot_test.go index 5dfae55..e5f3ba4 100644 --- a/internal/ratelimit/snapshot_test.go +++ b/internal/ratelimit/snapshot_test.go @@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) { "2001:db8::/64", } - limiter := ratelimit.New(ratelimit.Limits{}, tableSize) + limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) for _, i := range []int{2, 3, 0, 1} { limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole) } @@ -52,7 +52,10 @@ func TestLoadedCountsCarryOn(t *testing.T) { client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() - before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) + before := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerHour: limit}, + MaxClients: tableSize, + }) for range limit { wantCount(t, before, client, start, "") } @@ -60,7 +63,10 @@ func TestLoadedCountsCarryOn(t *testing.T) { // Loaded into a new limiter, as across a restart, the client has no // fresh allowance. later := start.Add(time.Minute) - after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) + after := ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{PerHour: limit}, + MaxClients: tableSize, + }) after.Load(before.Snapshot(), later) wantCount(t, after, client, later, hour) } @@ -71,7 +77,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() - limiter := ratelimit.New(ratelimit.Limits{}, tableSize) + limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter.Count(client, start, whole) limiter.CountBytes(client, start, 5, whole) limiter.CountRefusal(client, start, limit) @@ -80,7 +86,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { loaded := func(now time.Time) ratelimit.Client { t.Helper() - after := ratelimit.New(ratelimit.Limits{}, tableSize) + after := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) after.Load(limiter.Snapshot(), now) return after.Snapshot()[0] @@ -142,7 +148,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) { addr = addr.Next() } - limiter := ratelimit.New(ratelimit.Limits{}, maxClients) + limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients}) limiter.Load(clients, midnight()) got := limiter.Snapshot() diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index c432c17..55aa26f 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -1,7 +1,7 @@ -// Package requestlog writes the lines sww prints on stdout: one +// Package requestlog writes the request log sww prints on stdout: one // JSON object per request, two from waf and a short one from rproxy, -// marked "type":"request", and the process's own messages as JSON lines marked -// "type":"process". +// marked "type":"request". Package processlog writes the process's own +// messages. package requestlog import ( @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "log/slog" "time" "sneak.berlin/go/smallwebwaf/internal/ratelimit" @@ -300,31 +299,3 @@ func Milliseconds(d time.Duration) float64 { time.Millisecond/time.Microsecond, ) } - -// NewProcessLogger returns the logger for the process's own messages: -// JSON lines on w, marked "type":"process", with the time in the same form -// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance. -// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe; -// the request lines Write writes are never held back. -func NewProcessLogger( - w io.Writer, - instanceName string, - level slog.Level, -) *slog.Logger { - handler := slog.NewJSONHandler(w, &slog.HandlerOptions{ - Level: level, - ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr { - if attr.Key == slog.TimeKey && len(groups) == 0 { - return slog.String( - slog.TimeKey, - FormatTime(attr.Value.Time()), - ) - } - - return attr - }, - }) - - return slog.New(handler). - With("type", "process", "instance", instanceName) -} diff --git a/internal/requestlog/requestlog_test.go b/internal/requestlog/requestlog_test.go index b14faac..72488fc 100644 --- a/internal/requestlog/requestlog_test.go +++ b/internal/requestlog/requestlog_test.go @@ -3,8 +3,6 @@ package requestlog_test import ( "bytes" "encoding/json" - "log/slog" - "slices" "strings" "testing" "time" @@ -134,88 +132,3 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) { t.Errorf("wrote %s, want %s", out.String(), want) } } - -func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) { - t.Parallel() - - var out bytes.Buffer - - requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo). - Info("starting", - "version", "v1") - - var fields map[string]any - - err := json.Unmarshal(out.Bytes(), &fields) - if err != nil { - t.Fatalf("decode %q: %v", out.String(), err) - } - - if fields["type"] != "process" || - fields["instance"] != "host1/blog" || - fields["msg"] != "starting" || - fields["level"] != "INFO" || - fields["version"] != "v1" { - t.Errorf("process line %v", fields) - } - - timeText, _ := fields["time"].(string) - - logged, err := time.Parse(time.RFC3339, timeText) - if err != nil || !strings.HasSuffix(timeText, "Z") || - len(timeText) != len("2006-01-02T15:04:05.000Z") || - time.Since(logged) > time.Minute { - t.Errorf( - "process line time %q, want now in UTC with milliseconds", - timeText, - ) - } -} - -func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) { - t.Parallel() - - levels := []slog.Level{ - slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError, - } - - for i, level := range levels { - t.Run(level.String(), func(t *testing.T) { - t.Parallel() - - var out bytes.Buffer - - processLog := requestlog.NewProcessLogger( - &out, - "host1/blog", - level, - ) - for _, at := range levels { - processLog.Log(t.Context(), at, "message") - } - - var got, want []string - - for line := range strings.Lines(out.String()) { - var fields struct { - Level string `json:"level"` - } - - err := json.Unmarshal([]byte(line), &fields) - if err != nil { - t.Fatalf("decode %q: %v", line, err) - } - - got = append(got, fields.Level) - } - - for _, written := range levels[i:] { - want = append(want, written.String()) - } - - if !slices.Equal(got, want) { - t.Errorf("lines at %v, want %v", got, want) - } - }) - } -} diff --git a/internal/rproxy/limits_test.go b/internal/rproxy/limits_test.go index 8035e4e..9abc6ee 100644 --- a/internal/rproxy/limits_test.go +++ b/internal/rproxy/limits_test.go @@ -18,7 +18,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config/configtest" - "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/waf" ) @@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) { server := rproxy.New(rproxy.Params{ Config: cfg, RequestLog: io.Discard, - ProcessLog: requestlog.NewProcessLogger( - io.Discard, - "test", - slog.LevelInfo, - ), + ProcessLog: processlog.New(processlog.Params{ + Output: io.Discard, + InstanceName: "test", + Level: slog.LevelInfo, + }), }) if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || diff --git a/internal/rproxy/rproxy_test.go b/internal/rproxy/rproxy_test.go index 74e8eae..6049ea8 100644 --- a/internal/rproxy/rproxy_test.go +++ b/internal/rproxy/rproxy_test.go @@ -21,7 +21,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config/configtest" - "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/waf" ) @@ -370,11 +370,11 @@ func startServerLogging( server := rproxy.New(rproxy.Params{ Config: cfg, RequestLog: requestLog, - ProcessLog: requestlog.NewProcessLogger( - processLog, - "test", - slog.LevelInfo, - ), + ProcessLog: processlog.New(processlog.Params{ + Output: processLog, + InstanceName: "test", + Level: slog.LevelInfo, + }), }) listener, err := (&net.ListenConfig{}).Listen( diff --git a/internal/state/state_test.go b/internal/state/state_test.go index d1861b9..ae9a235 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -2448,7 +2448,7 @@ func newParams(dir string) state.Params { AttackBanDuration: 7 * 24 * time.Hour, MaxBans: 5000, }), - Limiter: ratelimit.New(ratelimit.Limits{}, 20000), + Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}), GeoJS: geojs.New(geojs.Params{Now: midnight}), Lists: reputation.New(reputation.Params{ BlocklistURLs: []string{ diff --git a/internal/sww/rproxy.go b/internal/sww/rproxy.go index 1f886fa..0b7b274 100644 --- a/internal/sww/rproxy.go +++ b/internal/sww/rproxy.go @@ -10,7 +10,7 @@ import ( "go.uber.org/fx" "sneak.berlin/go/smallwebwaf/internal/config" - "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/rproxy" ) @@ -23,11 +23,11 @@ import ( // setting for either. func RunRProxy(ctx context.Context, params Params) int { hostname, _ := os.Hostname() // "" when the host has no name to give - processLog := requestlog.NewProcessLogger( - params.Stdout, - hostname, - slog.LevelInfo, - ) + processLog := processlog.New(processlog.Params{ + Output: params.Stdout, + InstanceName: hostname, + Level: slog.LevelInfo, + }) cfg, err := config.RProxyFromEnvironment(params.Viper) if err != nil { diff --git a/internal/sww/waf.go b/internal/sww/waf.go index 4c175b7..ae3f13b 100644 --- a/internal/sww/waf.go +++ b/internal/sww/waf.go @@ -19,10 +19,10 @@ import ( "sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" + "sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/reputation" - "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/routewatch" "sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/state" @@ -43,8 +43,11 @@ const remoteLogStopTimeout = 2 * time.Second func RunWAF(ctx context.Context, params Params) int { // Until the settings are read, the one message is an invalid setting's // error, which every SWW_WAF_LOG_LEVEL lets through. - processLog := requestlog.NewProcessLogger(params.Stdout, - config.InstanceName(params.Viper), slog.LevelError) + processLog := processlog.New(processlog.Params{ + Output: params.Stdout, + InstanceName: config.InstanceName(params.Viper), + Level: slog.LevelError, + }) cfg, err := config.WAFFromEnvironment(params.Viper) if err != nil { @@ -63,11 +66,11 @@ func RunWAF(ctx context.Context, params Params) int { stdout = io.MultiWriter(params.Stdout, remote) } - processLog = requestlog.NewProcessLogger( - stdout, - cfg.InstanceName, - cfg.LogLevel, - ) + processLog = processlog.New(processlog.Params{ + Output: stdout, + InstanceName: cfg.InstanceName, + Level: cfg.LogLevel, + }) if remote != nil { stopSending := startSending(ctx, remote, processLog) @@ -238,14 +241,17 @@ func newLookupFile( // newLimiter returns the table of clients, with the rate and byte limits. func newLimiter(cfg *config.WAF) *ratelimit.Limiter { - return ratelimit.New(ratelimit.Limits{ - PerMinute: cfg.RateLimitPerMinute, - PerHour: cfg.RateLimitPerHour, - PerDay: cfg.RateLimitPerDay, - BytesPerMinute: cfg.BytesLimitPerMinute, - BytesPerHour: cfg.BytesLimitPerHour, - BytesPerDay: cfg.BytesLimitPerDay, - }, cfg.MaxTrackedClients) + return ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{ + PerMinute: cfg.RateLimitPerMinute, + PerHour: cfg.RateLimitPerHour, + PerDay: cfg.RateLimitPerDay, + BytesPerMinute: cfg.BytesLimitPerMinute, + BytesPerHour: cfg.BytesLimitPerHour, + BytesPerDay: cfg.BytesLimitPerDay, + }, + MaxClients: cfg.MaxTrackedClients, + }) } // newLedger returns the ledger of the bans, with the settings for them. diff --git a/internal/waf/proxy_test.go b/internal/waf/proxy_test.go index f5eeb27..bc086a8 100644 --- a/internal/waf/proxy_test.go +++ b/internal/waf/proxy_test.go @@ -24,6 +24,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" + "sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" @@ -398,11 +399,11 @@ func startRProxy( server := rproxy.New(rproxy.Params{ Config: cfg, RequestLog: io.Discard, - ProcessLog: requestlog.NewProcessLogger( - io.Discard, - "rproxy", - slog.LevelInfo, - ), + ProcessLog: processlog.New(processlog.Params{ + Output: io.Discard, + InstanceName: "rproxy", + Level: slog.LevelInfo, + }), }) listener, err := (&net.ListenConfig{}).Listen( @@ -471,11 +472,11 @@ func newWAFWith( } out := &output{} - processLog := requestlog.NewProcessLogger( - out, - cfg.InstanceName, - cfg.LogLevel, - ) + processLog := processlog.New(processlog.Params{ + Output: out, + InstanceName: cfg.InstanceName, + Level: cfg.LogLevel, + }) ruleFiles, err := rules.Load(rules.Params{ Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog, @@ -561,14 +562,17 @@ func addCounters(params *waf.Params) { cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName, ) - params.Limiter = ratelimit.New(ratelimit.Limits{ - PerMinute: cfg.RateLimitPerMinute, - PerHour: cfg.RateLimitPerHour, - PerDay: cfg.RateLimitPerDay, - BytesPerMinute: cfg.BytesLimitPerMinute, - BytesPerHour: cfg.BytesLimitPerHour, - BytesPerDay: cfg.BytesLimitPerDay, - }, cfg.MaxTrackedClients) + params.Limiter = ratelimit.New(ratelimit.Params{ + Limits: ratelimit.Limits{ + PerMinute: cfg.RateLimitPerMinute, + PerHour: cfg.RateLimitPerHour, + PerDay: cfg.RateLimitPerDay, + BytesPerMinute: cfg.BytesLimitPerMinute, + BytesPerHour: cfg.BytesLimitPerHour, + BytesPerDay: cfg.BytesLimitPerDay, + }, + MaxClients: cfg.MaxTrackedClients, + }) params.Ledger = bans.New(bans.Params{ LimitBanDuration: cfg.LimitBanDuration, LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,