AS number and country looked up for every client (closes #95)
check / check (push) Canceled after 0s

GeoJS's geo.json is asked about every new visitor unless
SWWAF_LOOKUP_SOURCE is off. A request waits for its client's first
answer only while a country list or SWWAF_ADD_LOOKUP_HEADERS needs it;
otherwise the answer reaches the client's history and ban notes when it
comes. The AS number and name go beside the country in the request log,
history, ban notes, alerts and lookups.json, with metrics by AS number;
64512 counts as unknown.

Judgement call: AS numbers are written AS64496, as SPEC's settings write them.
Judgement call: SWWAF_LOOKUP_TIMEOUT is added, default 1s, and cannot be off.
Judgement call: a client's own X-Client-* headers are removed only while SWWAF_ADD_LOOKUP_HEADERS is set.

Model: opus-5-5
This commit is contained in:
2026-10-07 05:28:02 +00:00
parent f35cbd01cf
commit d4f5b3e404
29 changed files with 1371 additions and 436 deletions
+3 -2
View File
@@ -137,8 +137,9 @@ type Params struct {
}
// Alert is one alert, as the webhook is sent it and alerts.json holds it,
// with the fields of the "Alert webhook schema" section of SPEC.md. ASN
// and ASName are empty until AS numbers are looked up.
// with the fields of the "Alert webhook schema" section of SPEC.md. ASN,
// ASName and Country are, for a ban, the client's as the ban's notes give
// them.
//
//nolint:tagliatelle // SPEC.md's alert webhook schema names its fields in snake_case
type Alert struct {
+27 -1
View File
@@ -91,7 +91,11 @@ func (b Ban) ActiveAt(now time.Time) bool {
//
//nolint:tagliatelle // the state files use snake_case, as the request log does
type Notes struct {
// Country is the client's country, when it was looked up.
// ASN, ASName and Country are the client's AS number, AS name and
// country, when they were looked up: when the request that caused the
// ban was made, or when GeoJS answered about the client afterwards.
ASN string `json:"asn"`
ASName string `json:"as_name"`
Country string `json:"country"`
// Limit, Window and Count are, for a ban for a broken limit, the limit
// that was broken, its window, "minute", "hour" or "day", and the
@@ -411,6 +415,28 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban {
return slices.Clone(*bans)
}
// AddLookup gives the notes of netblock's bans that have no AS number, AS
// name or country yet those of a client in it, as GeoJS answered about
// it. It is not a request from netblock, and leaves when it was last seen
// unchanged. It does not have bans.json written at once: the notes are
// written with its next write, as the counts in them are.
func (l *Ledger) AddLookup(netblock netip.Prefix, asn, asName, country string) {
l.mu.Lock()
defer l.mu.Unlock()
bans, found := l.netblocks.Peek(netblock)
if !found {
return
}
for i := range *bans {
notes := &(*bans)[i].Notes
if notes.ASN == "" && notes.ASName == "" && notes.Country == "" {
notes.ASN, notes.ASName, notes.Country = asn, asName, country
}
}
}
// Made returns how many bans for cause have been made since the start:
// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an
// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts
+39
View File
@@ -454,6 +454,45 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
}
}
func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultRules())
netblock := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32")
// A ban made with the client's lookup, one made before it came, after
// the first ended, and one on another netblock.
ledger.BanForLimit(netblock, midnight(), bans.Notes{
ASN: "AS64497", ASName: "Other Net", Country: "FR",
})
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
ledger.BanForLimit(other, midnight(), bans.Notes{})
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
held := ledger.Bans(netblock)
if len(held) != 2 {
t.Fatalf("%s has %d bans, want 2", netblock, len(held))
}
for i, want := range []bans.Notes{
{ASN: "AS64497", ASName: "Other Net", Country: "FR"},
{ASN: "AS64496", ASName: "Example Net", Country: "DE"},
} {
got := held[i].Notes
if got.ASN != want.ASN || got.ASName != want.ASName || got.Country != want.Country {
t.Errorf("ban %d's notes give %q, %q and %q, want %q, %q and %q", i+1,
got.ASN, got.ASName, got.Country, want.ASN, want.ASName, want.Country)
}
}
if notes := ledger.Bans(other)[0].Notes; notes.ASN != "" || notes.Country != "" {
t.Errorf("the ban on %s has %q and %q, want neither",
other, notes.ASN, notes.Country)
}
}
// defaultRules are the rules at the settings' defaults.
func defaultRules() bans.Rules {
return bans.Rules{
+58 -9
View File
@@ -91,6 +91,15 @@ type Config struct {
// limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_PATHS).
// Each starts with /.
RateLimitExemptPaths []string
// LookupSource is where each client's AS number and country are
// looked up (SWWAF_LOOKUP_SOURCE): geojs, or off for nowhere. A request
// waits up to LookupTimeout for its client's first answer while a
// setting needs it (SWWAF_LOOKUP_TIMEOUT), which cannot be off.
// AddLookupHeaders is true when the app is passed the client's AS
// number and country in headers (SWWAF_ADD_LOOKUP_HEADERS).
LookupSource string
LookupTimeout time.Duration
AddLookupHeaders bool
// DeniedCountries are the countries whose clients are refused
// (SWWAF_DENIED_COUNTRIES). ExclusivelyAllowedCountries, when not
// empty, are the only countries whose clients are let through
@@ -138,8 +147,8 @@ type Config struct {
AdminToken string
// MetricsToken is the bearer token a scraper sends for the metrics
// (SWWAF_METRICS_TOKEN), "" while it is unset and the metrics are off.
// MetricsTopN is how many countries get series of their own in the
// metrics (SWWAF_METRICS_TOP_N).
// MetricsTopN is how many AS numbers and how many countries get series
// of their own in the metrics (SWWAF_METRICS_TOP_N).
MetricsToken string
MetricsTopN int
// RulesDir is the directory of the rule files (SWWAF_RULES_DIR), read
@@ -233,6 +242,8 @@ var (
"is taken out of every request by Go's HTTP server, so it can never " +
"be logged")
errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too")
errNotLookupSource = errors.New("is not geojs or off")
errNeedsLookups = errors.New("it needs each client looked up")
errNotOver4K = errors.New("is not a size of more than 4K, such as 32K")
errNotDurationAboveZero = errors.New(
"is not a duration above zero, such as 1h or 7d")
@@ -300,6 +311,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"),
RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"),
RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""),
LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"),
LookupTimeout: env.durationNotOff("SWWAF_LOOKUP_TIMEOUT", "1s"),
AddLookupHeaders: env.boolean("SWWAF_ADD_LOOKUP_HEADERS", "false"),
DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""),
ExclusivelyAllowedCountries: env.countries(
"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
@@ -339,13 +353,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
cfg.InstanceName, cfg.LogRemoteURL != nil)
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
for _, country := range cfg.ExclusivelyAllowedCountries {
if slices.Contains(cfg.DeniedCountries, country) {
env.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
fmt.Errorf("%q %w", country, errOnBothLists))
}
}
env.checkCountriesAndLookups(cfg)
if env.err != nil {
return nil, env.err
@@ -547,6 +555,47 @@ func (e *environment) countries(name, defaultValue string) []string {
return countries
}
// lookupSource reads the setting that is where clients are looked up:
// geojs, or off.
func (e *environment) lookupSource(name, defaultValue string) string {
source := e.value(name, defaultValue)
if source != "geojs" && source != off {
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
}
return source
}
// checkCountriesAndLookups refuses a country on both country lists, and,
// while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked
// up: the country lists and SWWAF_ADD_LOOKUP_HEADERS.
func (e *environment) checkCountriesAndLookups(cfg *Config) {
for _, country := range cfg.ExclusivelyAllowedCountries {
if slices.Contains(cfg.DeniedCountries, country) {
e.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
fmt.Errorf("%q %w", country, errOnBothLists))
}
}
if cfg.LookupSource != off {
return
}
for _, setting := range []struct {
name string
set bool
}{
{"SWWAF_DENIED_COUNTRIES", len(cfg.DeniedCountries) > 0},
{"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
{"SWWAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
} {
if setting.set {
e.check(setting.name, fmt.Errorf("is set while SWWAF_LOOKUP_SOURCE is off; %w",
errNeedsLookups))
}
}
}
// headerNames reads a setting that is a list of header names, and
// returns them in lower case.
func (e *environment) headerNames(name, defaultValue string) []string {
+74
View File
@@ -40,6 +40,9 @@ const (
rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR"
rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY"
rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
lookupSource = "SWWAF_LOOKUP_SOURCE"
lookupTimeout = "SWWAF_LOOKUP_TIMEOUT"
addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS"
deniedCountries = "SWWAF_DENIED_COUNTRIES"
allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
banResponse = "SWWAF_BAN_RESPONSE"
@@ -117,6 +120,12 @@ const (
// off switches a timeout, a size limit or a rate limit off.
const off = "off"
// enabled is true, as a setting's value.
const enabled = "true"
// defaultLookupSource is the default of SWWAF_LOOKUP_SOURCE.
const defaultLookupSource = "geojs"
// environment is a set of environment variables, for FromEnvironment.
type environment map[string]string
@@ -173,6 +182,9 @@ func TestDefaults(t *testing.T) {
RulesDir: "/etc/smallwebwaf/rules.d",
RulesEnabled: true,
})
wantLookupSettings(t, cfg, config.Config{
LookupSource: defaultLookupSource, LookupTimeout: time.Second,
})
if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" {
t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL)
@@ -773,6 +785,50 @@ func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
}
}
func TestLookupSettingsAsSet(t *testing.T) {
t.Parallel()
cfg := fromEnvironment(t, environment{
lookupTimeout: "500ms", addLookupHeaders: enabled,
})
wantLookupSettings(t, cfg, config.Config{
LookupSource: defaultLookupSource, LookupTimeout: 500 * time.Millisecond,
AddLookupHeaders: true,
})
cfg = fromEnvironment(t, environment{lookupSource: off})
wantLookupSettings(t, cfg, config.Config{
LookupSource: off, LookupTimeout: time.Second,
})
}
func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
t.Parallel()
for name, value := range map[string]string{
deniedCountries: "kp",
allowedCountries: "de",
addLookupHeaders: enabled,
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
_, err := config.FromEnvironment(environment{
lookupSource: off, name: value,
}.lookupEnv)
if err == nil || !strings.HasPrefix(err.Error(), name+": ") ||
!strings.Contains(err.Error(), lookupSource+" is off") {
t.Errorf("error %v, want one naming %s and %s", err, name, lookupSource)
}
})
}
// Set empty, the country lists need nothing looked up.
fromEnvironment(t, environment{
lookupSource: off, deniedCountries: "", allowedCountries: "",
})
}
func TestSizesAndOff(t *testing.T) {
t.Parallel()
@@ -894,6 +950,9 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
{rateLimitPerHour, "1.5"},
{rateLimitPerDay, "-1"}, {rateLimitPerDay, "lots"},
{rateLimitExemptPaths, "/assets/,,/static/"},
{lookupSource, "file"}, {lookupSource, "GeoJS"}, {lookupSource, ""},
{lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"},
{addLookupHeaders, "yes"},
{deniedCountries, "nk"},
{deniedCountries, "kp,,ir"},
{deniedCountries, "prk"},
@@ -1181,6 +1240,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
rateLimitPerHour: "10000",
rateLimitPerDay: "50000",
rateLimitExemptPaths: "",
lookupSource: defaultLookupSource,
lookupTimeout: "1s",
addLookupHeaders: "false",
deniedCountries: "",
allowedCountries: "",
banResponse: "403",
@@ -1242,6 +1304,18 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) {
wantBanSettings(t, got, want)
}
// wantLookupSettings checks the settings for lookups.
func wantLookupSettings(t *testing.T, got *config.Config, want config.Config) {
t.Helper()
if got.LookupSource != want.LookupSource || got.LookupTimeout != want.LookupTimeout ||
got.AddLookupHeaders != want.AddLookupHeaders {
t.Errorf("lookups from %q, waited for %s, headers %t; want %q, %s, %t",
got.LookupSource, got.LookupTimeout, got.AddLookupHeaders,
want.LookupSource, want.LookupTimeout, want.AddLookupHeaders)
}
}
// wantBanSettings checks the settings for bans, the state files, the
// metrics and the rule files.
func wantBanSettings(t *testing.T, got *config.Config, want config.Config) {
+124 -64
View File
@@ -1,7 +1,7 @@
// Package lookup looks up each client's country through the GeoJS web
// service, and keeps the answers in memory, for at most 100,000 clients
// and for 7 days each. The answers are written to lookups.json and read
// from it by the state package.
// Package lookup looks up each client's AS number and country through
// the GeoJS web service, and keeps the answers in memory, for at most
// 100,000 clients and for 7 days each. The answers are written to
// lookups.json and read from it by the state package.
package lookup
import (
@@ -14,6 +14,7 @@ import (
"net/http"
"net/netip"
"slices"
"strconv"
"strings"
"sync"
"time"
@@ -23,9 +24,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/metrics"
)
// URL is GeoJS's country endpoint. Asked about several addresses at once,
// comma separated in its ip parameter, it answers with a list.
const URL = "https://get.geojs.io/v1/ip/country.json"
// URL is GeoJS's endpoint for an address's place and network. Asked about
// several addresses at once, comma separated in its ip parameter, it
// answers with a list.
const URL = "https://get.geojs.io/v1/ip/geo.json"
const (
// keepFor is how long an answer is used instead of asking GeoJS again.
@@ -40,9 +42,8 @@ const (
maxWaiting = 10000
// maxPerRequest is how many addresses one request to GeoJS asks about.
maxPerRequest = 200
// timeout is how long a new client waits for its answer, and how long
// a request to GeoJS may take before it is abandoned.
timeout = time.Second
// unknownASN is the AS number GeoJS gives when it knows none.
unknownASN = 64512
// After a failure GeoJS is not asked again for a second, and for
// retryDelayFactor times as long after each further failure in a row,
// up to five minutes.
@@ -62,6 +63,16 @@ var (
type Params struct {
// URL is where GeoJS is asked, normally URL.
URL string
// Timeout is how long a request waits for its client's first answer,
// and how long a request to GeoJS may take before it is abandoned
// (SWWAF_LOOKUP_TIMEOUT).
Timeout time.Duration
// Wait is true when a setting needs each request's answer before the
// request goes on. Otherwise no request waits for one.
Wait bool
// Answered, unless nil, is given each answer GeoJS gives, once it is
// kept.
Answered func(Answer)
// Now tells the time, normally time.Now.
Now func() time.Time
// ProcessLog receives GeoJS's failures.
@@ -73,11 +84,14 @@ type Params struct {
Alerts *alerts.Queue
}
// GeoJS looks up clients' countries through GeoJS. At most one request
// to GeoJS is under way at a time, and it asks about every client waiting,
// up to maxPerRequest. It is safe for concurrent use.
// GeoJS looks up clients' AS numbers and countries through GeoJS. At most
// one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct {
url string
timeout time.Duration
wait bool
answered func(Answer)
now func() time.Time
processLog *slog.Logger
metrics *metrics.Metrics
@@ -100,10 +114,16 @@ type GeoJS struct {
}
// Answer is what GeoJS said about a client, as lookups.json holds it: its
// country, "" when GeoJS cannot place it, when GeoJS said so, and when
// the answer was last used.
// AS number, such as AS64496, and the AS's name, both "" when GeoJS knows
// no AS number for it; its country, "" when GeoJS cannot place it; when
// GeoJS said so, and when the answer was last used. The zero Answer is
// that of a client with no answer.
//
//nolint:tagliatelle // the state files use snake_case, as the request log does
type Answer struct {
Client netip.Prefix `json:"client"`
ASN string `json:"asn"`
ASName string `json:"as_name"`
Country string `json:"country"`
Answered time.Time `json:"answered"`
Used time.Time `json:"used"`
@@ -128,6 +148,9 @@ func New(params Params) *GeoJS {
return &GeoJS{
url: params.URL,
timeout: params.Timeout,
wait: params.Wait,
answered: params.Answered,
now: params.Now,
processLog: params.ProcessLog,
metrics: params.Metrics,
@@ -142,23 +165,23 @@ func New(params Params) *GeoJS {
}
}
// Country returns the country GeoJS places client in, as a two-letter
// code in capitals, or "" when the country cannot be found: GeoJS cannot
// place the client, or has not answered in time. An answer is kept for 7
// days. Without one, a client waits up to timeout for it, unless it has
// gone without one before; until GeoJS answers, the client is asked about
// again in the background. ctx is the context of the client's request,
// and ends the wait when it ends.
// LookUp returns the answer GeoJS gave about client, with its country as
// a two-letter code in capitals, or the zero Answer when there is none
// yet. An answer is kept for 7 days. Without one, the client is asked
// about in the background, and, while Wait is set, the request waits up
// to Timeout for the answer, unless the client has gone without one
// before. ctx is the context of the client's request, and ends the wait
// when it ends.
//
// GeoJS is asked about the client's first address, which is the client's
// own address for IPv4, and an address in the same place for an IPv6 /64.
func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
country, asked := g.answerOrWait(ctx, client)
func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer {
answer, asked := g.answerOrWait(ctx, client)
if asked == nil {
return country
return answer
}
timer := time.NewTimer(timeout)
timer := time.NewTimer(g.timeout)
defer timer.Stop()
select {
@@ -170,7 +193,7 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
g.mu.Lock()
defer g.mu.Unlock()
country, found := g.kept(client)
answer, found := g.kept(client)
if !found {
g.metrics.GeoJSUnanswered.Inc()
}
@@ -180,7 +203,15 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
w.late = true
}
return country
return answer
}
// Kept returns client's answer, if one is kept, without asking GeoJS.
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
g.mu.Lock()
defer g.mu.Unlock()
return g.kept(client)
}
// Snapshot returns every answer kept, sorted by client, as lookups.json
@@ -232,13 +263,13 @@ func (g *GeoJS) Load(answers []Answer) {
// nil when there is nothing to wait for.
func (g *GeoJS) answerOrWait(
ctx context.Context, client netip.Prefix,
) (string, <-chan struct{}) {
) (Answer, <-chan struct{}) {
g.mu.Lock()
defer g.mu.Unlock()
country, found := g.kept(client)
answer, found := g.kept(client)
if found {
return country, nil
return answer, nil
}
w, waiting := g.waiting[client]
@@ -249,10 +280,14 @@ func (g *GeoJS) answerOrWait(
g.ask(ctx)
if !g.wait {
return Answer{}, nil // the answer is not needed before the request goes on
}
if w == nil {
g.metrics.GeoJSUnanswered.Inc()
return "", nil // too many clients wait already
return Answer{}, nil // too many clients wait already
}
if !g.asking {
@@ -263,25 +298,25 @@ func (g *GeoJS) answerOrWait(
if w.late {
g.metrics.GeoJSUnanswered.Inc()
return "", nil
return Answer{}, nil
}
return "", w.asked
return Answer{}, w.asked
}
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (string, bool) {
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.now()
kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor {
return "", false
return Answer{}, false
}
kept.Used = now
return kept.Country, true
return *kept, true
}
// ask starts asking GeoJS about the waiting clients, unless a request to
@@ -299,7 +334,8 @@ func (g *GeoJS) ask(ctx context.Context) {
}
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
// time, until none is left or GeoJS fails.
// time, until none is left or GeoJS fails. Each answer kept is given to
// Answered, outside the lock, since Answered takes locks of its own.
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
for {
clients := g.nextClients()
@@ -307,8 +343,16 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
return
}
countries, err := g.request(ctx, clients)
if !g.keep(clients, countries, err) {
given, err := g.request(ctx, clients)
kept, answered := g.keep(clients, given, err)
if g.answered != nil {
for _, answer := range kept {
g.answered(answer)
}
}
if !answered {
return
}
}
@@ -340,32 +384,35 @@ func (g *GeoJS) nextClients() []netip.Prefix {
return clients
}
// keep notes how a request to GeoJS about clients ended, and reports
// whether GeoJS answered about all of them. Each client whose address
// GeoJS's answer names gets its answer, with no country when GeoJS gave
// none. An answer that leaves an address out is a failure. After a
// failure GeoJS is left alone for a while, and every client still waiting
// stops waiting and is asked about once GeoJS is asked again.
// keep notes how a request to GeoJS about clients ended, given being the
// answer for each address GeoJS's answer names. It returns the answers it
// kept, and reports whether GeoJS answered about all of the clients. Each
// client whose address GeoJS's answer names gets its answer. An answer
// that leaves an address out is a failure. After a failure GeoJS is left
// alone for a while, and every client still waiting stops waiting and is
// asked about once GeoJS is asked again.
func (g *GeoJS) keep(
clients []netip.Prefix, countries map[netip.Addr]string, err error,
) bool {
clients []netip.Prefix, given map[netip.Addr]Answer, err error,
) ([]Answer, bool) {
g.mu.Lock()
defer g.mu.Unlock()
now := g.now()
kept := make([]Answer, 0, len(clients))
leftOut := 0
for _, client := range clients {
country, named := countries[client.Addr()]
answer, named := given[client.Addr()]
if !named {
leftOut++
continue
}
g.answers.Add(client, &Answer{
Client: client, Country: country, Answered: now, Used: now,
})
answer.Client, answer.Answered, answer.Used = client, now, now
g.answers.Add(client, &answer)
kept = append(kept, answer)
close(g.waiting[client].asked)
delete(g.waiting, client)
}
@@ -400,26 +447,28 @@ func (g *GeoJS) keep(
},
})
return false
return kept, false
}
g.retryDelay = 0
return true
return kept, true
}
// request asks GeoJS about clients in one request, and returns the
// country it gave, in capitals, for each address its answer names.
// request asks GeoJS about clients in one request, and returns the answer
// for each address GeoJS's answer names: its AS number and the AS's name,
// both "" for the AS number 64512, which GeoJS gives when it knows none,
// and its country, in capitals.
func (g *GeoJS) request(
ctx context.Context, clients []netip.Prefix,
) (map[netip.Addr]string, error) {
) (map[netip.Addr]Answer, error) {
addrs := make([]string, 0, len(clients))
for _, client := range clients {
addrs = append(addrs, client.Addr().String())
}
ctx, cancel := context.WithTimeout(ctx, timeout)
ctx, cancel := context.WithTimeout(ctx, g.timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.url, http.NoBody)
@@ -446,9 +495,12 @@ func (g *GeoJS) request(
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
}
//nolint:tagliatelle // GeoJS's own names
var answers []struct {
IP string `json:"ip"`
Country string `json:"country"`
IP string `json:"ip"`
ASN int64 `json:"asn"`
ASName string `json:"organization_name"`
CountryCode string `json:"country_code"`
}
err = json.NewDecoder(io.LimitReader(res.Body, maxResponseBytes)).Decode(&answers)
@@ -456,14 +508,22 @@ func (g *GeoJS) request(
return nil, fmt.Errorf("read GeoJS's answer: %w", err)
}
countries := make(map[netip.Addr]string, len(answers))
given := make(map[netip.Addr]Answer, len(answers))
for _, item := range answers {
addr, err := netip.ParseAddr(item.IP)
if err == nil {
countries[addr] = strings.ToUpper(item.Country)
if err != nil {
continue
}
answer := Answer{Country: strings.ToUpper(item.CountryCode)}
if item.ASN != 0 && item.ASN != unknownASN {
answer.ASN = "AS" + strconv.FormatInt(item.ASN, 10)
answer.ASName = item.ASName
}
given[addr] = answer
}
return countries, nil
return given, nil
}
+121 -12
View File
@@ -23,9 +23,13 @@ import (
const (
// germany is where the stand-in for GeoJS places every address but
// unplaced.
germany = "DE"
// unplaced is the address it cannot place.
// unplaced, and asNumber, kept as asn, and asName the AS it gives them.
germany = "DE"
asNumber = 64496
asn = "AS64496"
asName = "Example Net"
// unplaced is the address it cannot place, for which it gives the AS
// number 64512 and the AS name Unknown, as GeoJS does.
unplaced = "192.0.2.1"
// leftOut is the address it leaves out of its answer when
// answeringWithoutLeftOut.
@@ -83,7 +87,7 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
var earlier sync.WaitGroup
earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) })
earlier.Go(func() { g.LookUp(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) })
defer earlier.Wait()
waitForRequests(t, geojs, 1)
@@ -187,6 +191,96 @@ func TestCountryIsKeptInCapitals(t *testing.T) {
})
}
func TestAnswerHoldsTheASNumberTheASNameAndTheCountry(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
_, clock, g := start()
placed := netip.MustParsePrefix("203.0.113.9/32")
notPlaced := netip.MustParsePrefix(unplaced + "/32")
now := clock.Now()
// For the client it cannot place, GeoJS gives the AS number 64512
// and the AS name Unknown, which count as unknown.
for client, want := range map[netip.Prefix]lookup.Answer{
placed: {
Client: placed, ASN: asn, ASName: asName, Country: germany,
Answered: now, Used: now,
},
notPlaced: {Client: notPlaced, Answered: now, Used: now},
} {
got := g.LookUp(t.Context(), client)
if got != want {
t.Errorf("answer for %s\n%+v\nwant\n%+v", client, got, want)
}
}
})
}
func TestWithoutWaitTheRequestGoesOnAtOnceAndTheAnswerIsGivenWhenItComes(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
var (
mu sync.Mutex
given []lookup.Answer
)
geojs := &standIn{answers: answeringSlowly}
clock := newClock()
m := metrics.New(1, "app")
g := lookup.New(lookup.Params{
URL: lookup.URL,
Timeout: timeout,
Answered: func(answer lookup.Answer) {
mu.Lock()
defer mu.Unlock()
given = append(given, answer)
},
Now: clock.Now,
ProcessLog: slog.New(slog.DiscardHandler),
Metrics: m,
Alerts: alerts.New(alerts.Params{}),
})
g.SetTransport(geojs)
client := netip.MustParsePrefix("203.0.113.9/32")
// The request goes on at once, without an answer, and GeoJS is asked
// about the client, which it answers most of a second later.
began := time.Now()
got := g.LookUp(t.Context(), client)
if took := time.Since(began); took != 0 || got != (lookup.Answer{}) {
t.Errorf("waited %s for %+v, want no wait and no answer", took, got)
}
waitForRequests(t, geojs, 1)
wantAsked(t, geojs, 0, "203.0.113.9")
time.Sleep(timeout)
synctest.Wait()
now := clock.Now()
want := lookup.Answer{
Client: client, ASN: asn, ASName: asName, Country: germany,
Answered: now, Used: now,
}
mu.Lock()
if !slices.Equal(given, []lookup.Answer{want}) {
t.Errorf("answers given %+v, want only %+v", given, want)
}
mu.Unlock()
wantCountry(t, g, client, germany)
wantUnanswered(t, m, 0)
})
}
func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) {
t.Parallel()
@@ -197,6 +291,8 @@ func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) {
geojs := &standIn{answers: hanging}
g := lookup.New(lookup.Params{
URL: lookup.URL,
Timeout: timeout,
Wait: true,
Now: time.Now,
ProcessLog: slog.New(slog.NewTextHandler(&log, nil)),
Metrics: metrics.New(1, "app"),
@@ -401,6 +497,8 @@ func TestClientsWithoutAnAnswerAreCounted(t *testing.T) {
m := metrics.New(1, "app")
g := lookup.New(lookup.Params{
URL: lookup.URL,
Timeout: timeout,
Wait: true,
Now: time.Now,
ProcessLog: slog.New(slog.DiscardHandler),
Metrics: m,
@@ -494,21 +592,24 @@ func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
list := make([]map[string]string, 0, len(addrs))
list := make([]map[string]any, 0, len(addrs))
for _, addr := range addrs {
country := germany
item := map[string]any{
"ip": addr, "asn": asNumber, "organization_name": asName,
"country_code": germany,
}
switch {
case addr == unplaced:
country = ""
item = map[string]any{"ip": addr, "asn": 64512, "organization_name": "Unknown"}
case addr == leftOut && answers == answeringWithoutLeftOut:
continue
case answers == answeringInLowerCase:
country = strings.ToLower(germany)
item["country_code"] = strings.ToLower(germany)
}
list = append(list, map[string]string{"ip": addr, "country": country})
list = append(list, item)
}
var answer any = list
@@ -566,7 +667,8 @@ func (c *testClock) advance(d time.Duration) {
}
// start returns a stand-in for GeoJS that answers, a clock, and a GeoJS
// asking the stand-in by that clock.
// asking the stand-in by that clock, for which a request waits for its
// client's first answer.
func start() (*standIn, *testClock, *lookup.GeoJS) {
geojs, clock, g, _ := startWithAlerts()
@@ -578,7 +680,7 @@ func start() (*standIn, *testClock, *lookup.GeoJS) {
// cooldown, by the same clock.
func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) {
geojs := &standIn{}
clock := &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)}
clock := newClock()
queue := alerts.New(alerts.Params{
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
Events: alerts.Events(),
@@ -587,6 +689,8 @@ func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) {
})
g := lookup.New(lookup.Params{
URL: lookup.URL,
Timeout: timeout,
Wait: true,
Now: clock.Now,
ProcessLog: slog.New(slog.DiscardHandler),
Metrics: metrics.New(1, "app"),
@@ -597,6 +701,11 @@ func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) {
return geojs, clock, g, queue
}
// newClock returns a clock set to the start of a day.
func newClock() *testClock {
return &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)}
}
// newClients returns what returns a new IPv4 client each time it is
// called.
func newClients() func() netip.Prefix {
@@ -613,7 +722,7 @@ func newClients() func() netip.Prefix {
func wantCountry(t *testing.T, g *lookup.GeoJS, client netip.Prefix, want string) {
t.Helper()
got := g.Country(t.Context(), client)
got := g.LookUp(t.Context(), client).Country
if got != want {
t.Errorf("%s is in %q, want %q", client, got, want)
}
+5 -2
View File
@@ -26,8 +26,11 @@ func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
wantCountry(t, g, placed, germany)
want := []lookup.Answer{
{Client: notPlaced, Country: "", Answered: asked, Used: asked},
{Client: placed, Country: germany, Answered: asked, Used: asked.Add(time.Hour)},
{Client: notPlaced, Answered: asked, Used: asked},
{
Client: placed, ASN: asn, ASName: asName, Country: germany,
Answered: asked, Used: asked.Add(time.Hour),
},
}
if got := g.Snapshot(); !slices.Equal(got, want) {
t.Errorf("snapshot\n%+v\nwant\n%+v", got, want)
+155
View File
@@ -0,0 +1,155 @@
package metrics
import (
"sync"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// other is the label value under which the countries or AS numbers
// outside the busiest are counted.
const other = "other"
// busiest are the metrics by one thing the lookup finds of the client,
// its country or its AS number, for requests whose client's is known.
// The topN busiest countries or AS numbers, by their requests since the
// start, have series of their own, and the others are counted under
// other, so that there are never more than topN + 1 series. One that
// drops out of the busiest loses its series, and its next requests are
// counted under other; one that becomes one of them gets a series that
// counts from then on. Each series therefore only ever goes up.
type busiest struct {
topN int
requests *prometheus.CounterVec
requestBytes *prometheus.CounterVec
responseBytes *prometheus.CounterVec
// refused are, by country, the requests the country lists refused; nil
// by AS number.
refused *prometheus.CounterVec
mu sync.Mutex
// seen is each country's or AS number's requests since the start, by
// which they are ranked.
seen map[string]int64
// top are those with series of their own.
top map[string]bool
}
// newCountries returns the metrics by country, with series of their own
// for the topN busiest countries.
func newCountries(topN int) *busiest {
countries := newBusiest(topN, "country", "the client's country")
countries.refused = counterVec("smallwebwaf_country_list_refusals_total",
"Requests the country lists refused, by the client's country.",
[]string{"country"})
return countries
}
// newASNs returns the metrics by AS number, with series of their own for
// the topN busiest AS numbers.
func newASNs(topN int) *busiest {
return newBusiest(topN, "asn", "the client's AS number")
}
// newBusiest returns the metrics by label, which is described as
// description, with series of their own for the topN busiest values.
func newBusiest(topN int, label, description string) *busiest {
by := []string{label}
return &busiest{
topN: topN,
requests: counterVec("smallwebwaf_"+label+"_requests_total",
"Requests, by "+description+".", by),
requestBytes: counterVec("smallwebwaf_"+label+"_request_bytes_total",
"Request body bytes, by "+description+".", by),
responseBytes: counterVec("smallwebwaf_"+label+"_response_bytes_total",
"Response body bytes, by "+description+".", by),
seen: map[string]int64{},
top: map[string]bool{},
}
}
// Describe and Collect make the metrics a prometheus.Collector, so that
// they are registered together.
func (b *busiest) Describe(ch chan<- *prometheus.Desc) {
for _, vec := range b.vecs() {
vec.Describe(ch)
}
}
// Collect is the other half of prometheus.Collector, with Describe.
func (b *busiest) Collect(ch chan<- prometheus.Metric) {
for _, vec := range b.vecs() {
vec.Collect(ch)
}
}
// vecs returns the metrics: by AS number, those of requests and bytes; by
// country, the refusals by the country lists as well.
func (b *busiest) vecs() []*prometheus.CounterVec {
vecs := []*prometheus.CounterVec{b.requests, b.requestBytes, b.responseBytes}
if b.refused != nil {
vecs = append(vecs, b.refused)
}
return vecs
}
// add counts a request from its log line, whose client's country or AS
// number, value, is known.
func (b *busiest) add(value string, line *requestlog.Line) {
b.mu.Lock()
defer b.mu.Unlock()
b.seen[value]++
label := b.label(value)
b.requests.WithLabelValues(label).Inc()
b.requestBytes.WithLabelValues(label).Add(float64(line.RequestBytes))
b.responseBytes.WithLabelValues(label).Add(float64(line.ResponseBytes))
if b.refused != nil && line.Action == requestlog.ActionCountryDenied {
b.refused.WithLabelValues(label).Inc()
}
}
// label returns the label a request from value is counted under: value
// while it is one of the busiest, other while it is not. A value busier
// than the least busy of them takes its place, and that one's series are
// dropped.
func (b *busiest) label(value string) string {
if b.top[value] {
return value
}
if len(b.top) < b.topN {
b.top[value] = true
return value
}
least := ""
for top := range b.top {
if least == "" || b.seen[top] < b.seen[least] {
least = top
}
}
if b.seen[value] <= b.seen[least] {
return other
}
delete(b.top, least)
for _, vec := range b.vecs() {
vec.DeleteLabelValues(least)
}
b.top[value] = true
return value
}
-116
View File
@@ -1,116 +0,0 @@
package metrics
import (
"sync"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// other is the label under which the countries outside the busiest are
// counted.
const other = "other"
// countries are the metrics by the client's country, for requests whose
// client's country is known. The topN busiest countries, by their requests
// since the start, have series of their own, and the others are counted
// under other, so that there are never more than topN + 1 series. A
// country that drops out of the busiest loses its series, and its next
// requests are counted under other; one that becomes one of them gets a
// series that counts from then on. Each series therefore only ever goes
// up.
type countries struct {
topN int
requests *prometheus.CounterVec
requestBytes *prometheus.CounterVec
responseBytes *prometheus.CounterVec
// refused are the requests the country lists refused.
refused *prometheus.CounterVec
mu sync.Mutex
// seen is each country's requests since the start, by which the
// countries are ranked. GeoJS gives two-letter codes, so it holds at
// most a few hundred.
seen map[string]int64
// top are the countries with series of their own.
top map[string]bool
}
// newCountries returns the metrics by country, with series of their own
// for the topN busiest countries.
func newCountries(topN int) *countries {
byCountry := []string{"country"}
return &countries{
topN: topN,
requests: counterVec("smallwebwaf_country_requests_total",
"Requests, by the client's country.", byCountry),
requestBytes: counterVec("smallwebwaf_country_request_bytes_total",
"Request body bytes, by the client's country.", byCountry),
responseBytes: counterVec("smallwebwaf_country_response_bytes_total",
"Response body bytes, by the client's country.", byCountry),
refused: counterVec("smallwebwaf_country_list_refusals_total",
"Requests the country lists refused, by the client's country.",
byCountry),
seen: map[string]int64{},
top: map[string]bool{},
}
}
// add counts a request from its log line, whose country is known.
func (c *countries) add(line *requestlog.Line) {
c.mu.Lock()
defer c.mu.Unlock()
c.seen[line.Country]++
label := c.label(line.Country)
c.requests.WithLabelValues(label).Inc()
c.requestBytes.WithLabelValues(label).Add(float64(line.RequestBytes))
c.responseBytes.WithLabelValues(label).Add(float64(line.ResponseBytes))
if line.Action == requestlog.ActionCountryDenied {
c.refused.WithLabelValues(label).Inc()
}
}
// label returns the label a request from country is counted under: the
// country while it is one of the busiest, other while it is not. A
// country busier than the least busy of them takes its place, and that
// country's series are dropped.
func (c *countries) label(country string) string {
if c.top[country] {
return country
}
if len(c.top) < c.topN {
c.top[country] = true
return country
}
least := ""
for top := range c.top {
if least == "" || c.seen[top] < c.seen[least] {
least = top
}
}
if c.seen[country] <= c.seen[least] {
return other
}
delete(c.top, least)
for _, vec := range []*prometheus.CounterVec{
c.requests, c.requestBytes, c.responseBytes, c.refused,
} {
vec.DeleteLabelValues(least)
}
c.top[country] = true
return country
}
+16 -12
View File
@@ -36,12 +36,13 @@ type Metrics struct {
offences *prometheus.CounterVec
// ruleMatches are made by AddRules.
ruleMatches *prometheus.CounterVec
countries *countries
countries *busiest
asns *busiest
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
// that failed. GeoJSUnanswered are the requests whose client counted
// as coming from an unknown country because GeoJS had not answered
// about it in time.
// that failed. GeoJSUnanswered are the requests that needed their
// client's answer, for a setting that acts on it, and went on without
// it because GeoJS had not given it in time.
GeoJSRequests prometheus.Counter
GeoJSFailures prometheus.Counter
GeoJSUnanswered prometheus.Counter
@@ -55,8 +56,8 @@ type Metrics struct {
}
// New returns the metrics, with the Go runtime's and the process's own.
// topN is how many countries get series of their own
// (SWWAF_METRICS_TOP_N). Every metric carries instanceName
// topN is how many countries and how many AS numbers get series of their
// own (SWWAF_METRICS_TOP_N). Every metric carries instanceName
// (SWWAF_INSTANCE_NAME) as its label instance.
func New(topN int, instanceName string) *Metrics {
byStatus := []string{"status_class", "action"}
@@ -96,6 +97,7 @@ func New(topN int, instanceName string) *Metrics {
offences: counterVec("smallwebwaf_offences_total",
"Offences, by kind.", []string{"kind"}),
countries: newCountries(topN),
asns: newASNs(topN),
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
Name: "smallwebwaf_geojs_requests_total",
Help: "Requests to GeoJS.",
@@ -106,8 +108,8 @@ func New(topN int, instanceName string) *Metrics {
}),
GeoJSUnanswered: prometheus.NewCounter(prometheus.CounterOpts{
Name: "smallwebwaf_geojs_unanswered_total",
Help: "Requests whose client counted as coming from an unknown " +
"country because GeoJS had not answered about it in time.",
Help: "Requests that needed their client's answer from GeoJS and " +
"went on without it, because GeoJS had not given it in time.",
}),
stateFileWrites: counterVec("smallwebwaf_state_file_writes_total",
"Writes of each state file.", byFile),
@@ -129,9 +131,7 @@ func New(topN int, instanceName string) *Metrics {
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
m.requestDuration, m.upstreamDuration,
m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences,
m.countries.requests, m.countries.requestBytes, m.countries.responseBytes,
m.countries.refused,
m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.countries, m.asns,
m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered,
m.stateFileWrites, m.stateFileWriteFailures,
m.stateFileLastWrite, m.stateFileSize,
@@ -315,7 +315,11 @@ func (m *Metrics) RequestEnded(
}
if line.Country != "" {
m.countries.add(line)
m.countries.add(line.Country, line)
}
if line.ASN != "" {
m.asns.add(line.ASN, line)
}
}
+6
View File
@@ -64,6 +64,8 @@ func (rq *request) limitBroken(now time.Time) bool {
}
notes := bans.Notes{
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
Limit: hit.Limit,
Window: hit.Window,
@@ -103,6 +105,8 @@ func (rq *request) banForAttack(now time.Time, rule rules.Rule) {
}
notes := bans.Notes{
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
RuleID: rule.ID,
Target: rule.Target,
@@ -165,6 +169,8 @@ func (rq *request) alertBan(ban bans.Ban) {
Event: event,
Client: rq.client,
Netblock: ban.Netblock,
ASN: ban.Notes.ASN,
ASName: ban.Notes.ASName,
Country: ban.Notes.Country,
Reason: ban.Reason,
Detail: detail,
+5 -2
View File
@@ -281,6 +281,8 @@ func TestBanNotes(t *testing.T) {
Cause: bans.CauseLimit,
Reason: "requests per minute over the limit of 1",
Notes: bans.Notes{
ASN: asnDE,
ASName: asNameDE,
Country: "DE",
Limit: 1,
Window: minute,
@@ -362,8 +364,9 @@ func (c *clock) advance(d time.Duration) {
// startWithClock starts smallwebwaf in front of an app that answers 200,
// with the settings in env on top of trusting localhost's
// X-Forwarded-For, clients' countries looked up at geojsURL, and a clock
// set to midnight, the start of a bucket in every window.
// X-Forwarded-For, clients' AS numbers and countries looked up at
// geojsURL, and a clock set to midnight, the start of a bucket in every
// window.
func startWithClock(
t *testing.T, geojsURL string, env map[string]string,
) (*sender, *clock, *proxy.Server) {
+6 -26
View File
@@ -1,31 +1,17 @@
package proxy
import (
"context"
"net/netip"
"slices"
)
// countryDenied reports whether the country lists refuse the request.
// The client's country is looked up only while a list is set, and never
// for a client on a private, loopback or link-local address, which has
// no country. A client without a country, or whose country cannot be
// found, is refused only by SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES. ctx is
// the request's own context.
func (rq *request) countryDenied(ctx context.Context) bool {
// countryDenied reports whether the country lists refuse the request, by
// the client's country as it was looked up. A client without a country,
// or whose country cannot be found, is refused only by
// SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES.
func (rq *request) countryDenied() bool {
denied := rq.h.config.DeniedCountries
allowed := rq.h.config.ExclusivelyAllowedCountries
if len(denied) == 0 && len(allowed) == 0 {
return false
}
var country string
if hasCountry(rq.client) {
country = rq.h.geojs.Country(ctx, clientGroup(rq.client))
}
rq.line.Country = country
country := rq.line.Country
if slices.Contains(denied, country) {
return true
@@ -33,9 +19,3 @@ func (rq *request) countryDenied(ctx context.Context) bool {
return len(allowed) > 0 && !slices.Contains(allowed, country)
}
// hasCountry reports whether addr can be placed in a country: private,
// loopback and link-local addresses cannot.
func hasCountry(addr netip.Addr) bool {
return !addr.IsPrivate() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast()
}
+99 -34
View File
@@ -56,16 +56,21 @@ func TestCountryLists(t *testing.T) {
maps.Copy(env, tc.env)
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env)
for i, sent := range []struct{ client, country string }{
{fromDE, "DE"}, {fromKP, "KP"}, {unplaced, ""},
// The AS number GeoJS gives unplaced, 64512, counts as unknown.
for i, sent := range []struct{ client, asn, asName, country string }{
{fromDE, asnDE, asNameDE, "DE"}, {fromKP, asnKP, asNameKP, "KP"},
{unplaced, "", "", ""},
} {
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
req.Header.Set(forwardedFor, sent.client)
got := do(t, req)
line := out.requestLines(t, i+1)[i]
if line.Country != sent.country {
t.Errorf("log line has country %q, want %q", line.Country, sent.country)
if line.ASN != sent.asn || line.ASName != sent.asName ||
line.Country != sent.country {
t.Errorf("log line has %q, %q and %q, want %q, %q and %q",
line.ASN, line.ASName, line.Country,
sent.asn, sent.asName, sent.country)
}
if slices.Contains(tc.refused, sent.client) {
@@ -130,7 +135,7 @@ func TestRequestRefusedByCountryIsNotCounted(t *testing.T) {
return
}
answer := []map[string]string{{"ip": r.URL.Query().Get("ip"), "country": "DE"}}
answer := []geojsAnswer{{IP: r.URL.Query().Get("ip"), CountryCode: "DE"}}
err := json.NewEncoder(w).Encode(answer)
if err != nil {
@@ -174,20 +179,15 @@ func TestRequestRefusedByCountryIsNotCounted(t *testing.T) {
wantStatus(t, got, http.StatusOK)
}
func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) {
func TestPrivateAddressIsNeverLookedUp(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
env map[string]string
clients []string // "" sends no X-Forwarded-For: the client is 127.0.0.1
name string
env map[string]string
}{
{"no country list is set", nil, []string{fromKP, fromDE}},
{
"private, loopback and link-local addresses",
map[string]string{deniedCountries: "kp"},
[]string{"10.0.0.5", "192.168.1.9", "fd00::5", "", "169.254.0.9", "fe80::9"},
},
{"no setting needs the lookup", nil},
{"a country list is set", map[string]string{deniedCountries: "kp"}},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
@@ -198,7 +198,10 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) {
maps.Copy(env, tc.env)
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env)
for i, sent := range tc.clients {
// "" sends no X-Forwarded-For: the client is 127.0.0.1.
for i, sent := range []string{
"10.0.0.5", "192.168.1.9", "fd00::5", "", "169.254.0.9", "fe80::9",
} {
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
if sent != "" {
req.Header.Set(forwardedFor, sent)
@@ -209,15 +212,26 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) {
line := out.requestLines(t, i+1)[i]
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
country, present := line.fields["country"]
if !present || country != "" {
t.Errorf("log line for %q has country %v, want an empty one",
line.ClientIP, country)
for _, field := range []string{"asn", "as_name", "country"} {
value, present := line.fields[field]
if !present || value != "" {
t.Errorf("log line for %q has %s %v, want an empty one",
line.ClientIP, field, value)
}
}
}
if len(asked()) != 0 {
t.Errorf("GeoJS was asked about %v, want nothing", asked())
// GeoJS is asked about up to 200 waiting clients at once, so once it
// has been asked about fromDE, which comes last, it has been asked
// about every client before it that waited for an answer.
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
req.Header.Set(forwardedFor, fromDE)
wantStatus(t, do(t, req), http.StatusOK)
waitUntil(func() bool { return slices.Contains(asked(), fromDE) })
if got := asked(); !slices.Equal(got, []string{fromDE}) {
t.Errorf("GeoJS was asked about %v, want %s alone", got, fromDE)
}
})
}
@@ -264,18 +278,31 @@ func TestExclusiveListRefusesAPrivateAddressUnlessAllowed(t *testing.T) {
}
}
// startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP
// and no other address. It returns its URL, and what returns the
// addresses it has been asked about.
// startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP,
// each in an AS of its own, and no other address. It returns its URL, and
// what returns the addresses it has been asked about.
func startGeoJS(t *testing.T) (string, func() []string) {
t.Helper()
places := map[string]string{fromDE: "DE", fromKP: "KP"}
geojsURL, asked, release := startHeldGeoJS(t)
release()
var asked struct {
mu sync.Mutex
addrs []string
}
return geojsURL, asked
}
// startHeldGeoJS is startGeoJS for a stand-in that answers nothing until
// release is called. Each request to it waits until then.
func startHeldGeoJS(t *testing.T) (string, func() []string, func()) {
t.Helper()
var (
asked struct {
mu sync.Mutex
addrs []string
}
released = make(chan struct{})
once sync.Once
)
geojs := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
@@ -285,11 +312,11 @@ func startGeoJS(t *testing.T) (string, func() []string) {
asked.addrs = append(asked.addrs, addrs...)
asked.mu.Unlock()
answers := make([]map[string]string, 0, len(addrs))
<-released
answers := make([]geojsAnswer, 0, len(addrs))
for _, addr := range addrs {
answers = append(answers, map[string]string{
"ip": addr, "country": places[addr],
})
answers = append(answers, answerAbout(addr))
}
err := json.NewEncoder(w).Encode(answers)
@@ -299,10 +326,48 @@ func startGeoJS(t *testing.T) (string, func() []string) {
}))
t.Cleanup(geojs.Close)
release := func() { once.Do(func() { close(released) }) }
// Run before geojs.Close, which waits for every request to be answered.
t.Cleanup(release)
return geojs.URL, func() []string {
asked.mu.Lock()
defer asked.mu.Unlock()
return slices.Clone(asked.addrs)
}, release
}
// The AS numbers and names the stand-in for GeoJS gives fromDE and
// fromKP, as they are logged.
const (
asnDE = "AS64496"
asNameDE = "Example Net"
asnKP = "AS64511"
asNameKP = "Other Net"
)
// geojsAnswer is an answer of GeoJS about one address, with the fields
// smallwebwaf reads.
//
//nolint:tagliatelle // GeoJS's own names
type geojsAnswer struct {
IP string `json:"ip"`
ASN int `json:"asn"`
ASName string `json:"organization_name"`
CountryCode string `json:"country_code,omitempty"`
}
// answerAbout is what the stand-in for GeoJS answers about addr: for an
// address it cannot place, the AS number 64512 and the AS name Unknown
// with no country, as GeoJS does.
func answerAbout(addr string) geojsAnswer {
switch addr {
case fromDE:
return geojsAnswer{IP: addr, ASN: 64496, ASName: asNameDE, CountryCode: "DE"}
case fromKP:
return geojsAnswer{IP: addr, ASN: 64511, ASName: asNameKP, CountryCode: "KP"}
}
return geojsAnswer{IP: addr, ASN: 64512, ASName: "Unknown"}
}
+6 -2
View File
@@ -24,7 +24,9 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
start := clk.Now()
// Two let through, one over the limit, which bans the client, and one
// refused under that ban, for which the country is not looked up.
// refused under that ban, for which the client is not looked up. GeoJS
// answers about the client at its first request, and its later ones
// use that answer.
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
clk.advance(time.Second)
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
@@ -35,8 +37,10 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
want := ratelimit.History{
FirstSeen: start,
LastSeen: start.Add(2 * time.Second),
ASN: asnDE,
ASName: asNameDE,
Country: "DE",
LookedUp: start.Add(time.Second),
LookedUp: start,
Requests: 4,
Forwarded: 2,
Refused: 2,
+67
View File
@@ -0,0 +1,67 @@
package proxy
import (
"context"
"net/http"
"net/netip"
"sneak.berlin/go/smallwebwaf/internal/lookup"
)
// The headers in which the app is passed the client's AS number and
// country while SWWAF_ADD_LOOKUP_HEADERS is set. Go sends a header name in
// this form, so X-Client-ASN arrives as X-Client-Asn; header names are not
// case-sensitive.
const (
asnHeader = "X-Client-Asn"
countryHeader = "X-Client-Country"
)
// lookUp looks up the client's AS number and country, and notes them for
// the log line, unless SWWAF_LOOKUP_SOURCE is off or the client is on a
// private, loopback or link-local address, which no lookup can place.
// While a setting needs the answer, a new client's request waits for it.
// ctx is the request's own context.
func (rq *request) lookUp(ctx context.Context) {
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
return
}
answer := rq.h.geojs.LookUp(ctx, clientGroup(rq.client))
rq.lookedUp = true
rq.line.ASN = answer.ASN
rq.line.ASName = answer.ASName
rq.line.Country = answer.Country
}
// addLookup adds answer, GeoJS's answer about a client, to the client's
// history, and to the notes of the bans on its netblock that have no AS
// number, AS name or country yet.
func (h *handler) addLookup(answer lookup.Answer) {
h.limiter.AddLookup(answer.Client, answer.Answered,
answer.ASN, answer.ASName, answer.Country)
h.ledger.AddLookup(h.netblock(answer.Client.Addr()),
answer.ASN, answer.ASName, answer.Country)
}
// setLookupHeaders sets the headers in which the app is passed the
// client's AS number and country, leaving out one that is unknown. Any
// the client sent are removed, so that the app can believe them.
func setLookupHeaders(header http.Header, asn, country string) {
header.Del(asnHeader)
header.Del(countryHeader)
if asn != "" {
header.Set(asnHeader, asn)
}
if country != "" {
header.Set(countryHeader, country)
}
}
// canBePlaced reports whether a lookup can place addr: private, loopback
// and link-local addresses have no AS number or country.
func canBePlaced(addr netip.Addr) bool {
return !addr.IsPrivate() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast()
}
+214
View File
@@ -0,0 +1,214 @@
package proxy_test
import (
"net/http"
"net/netip"
"slices"
"sync"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// asnAndCountry is what a lookup gives a client: its AS number, AS name
// and country.
type asnAndCountry struct{ asn, asName, country string }
func TestEveryClientIsLookedUpWithoutWaitingWhileNoSettingNeedsIt(t *testing.T) {
t.Parallel()
// The stand-in for GeoJS answers only once released. A request that
// waited for it would wait an hour, and get no answer within
// waitLimit.
geojsURL, asked, release := startHeldGeoJS(t)
s, _, server := startWithClock(t, geojsURL, map[string]string{
lookupTimeout: "1h",
rateLimitPerMinute: "1",
})
// fromDE's second request breaks the limit and bans it, and fromKP
// comes too. None waits for GeoJS.
for _, line := range []logLine{
s.get(fromDE, http.StatusOK, requestlog.ActionForward),
s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited),
s.get(fromKP, http.StatusOK, requestlog.ActionForward),
} {
got := asnAndCountry{line.ASN, line.ASName, line.Country}
if got != (asnAndCountry{}) {
t.Errorf("log line has %+v before GeoJS answered, want nothing", got)
}
}
// Once GeoJS answers, each answer reaches the client's history, and
// fromDE's reaches the notes of its ban.
release()
netblock := netip.MustParsePrefix(fromDE + "/32")
waitUntil(func() bool {
return historyOf(t, server, fromDE).ASN != "" &&
historyOf(t, server, fromKP).ASN != "" &&
server.Ledger.Bans(netblock)[0].Notes.ASN != ""
})
de := asnAndCountry{asnDE, asNameDE, "DE"}
for addr, want := range map[string]asnAndCountry{
fromDE: de, fromKP: {asnKP, asNameKP, "KP"},
} {
h := historyOf(t, server, addr)
if got := (asnAndCountry{h.ASN, h.ASName, h.Country}); got != want {
t.Errorf("%s's history has %+v, want %+v", addr, got, want)
}
}
notes := server.Ledger.Bans(netblock)[0].Notes
if got := (asnAndCountry{notes.ASN, notes.ASName, notes.Country}); got != de {
t.Errorf("the ban's notes have %+v, want %+v", got, de)
}
// GeoJS was asked about each client once, fromKP after fromDE, whose
// request was under way when fromKP came.
if got := asked(); !slices.Equal(got, []string{fromDE, fromKP}) {
t.Errorf("GeoJS was asked about %v, want %s and %s", got, fromDE, fromKP)
}
}
func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(t *testing.T) {
t.Parallel()
geojsURL, _ := startGeoJS(t)
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)}
addr, out, server, queue := startProxyWithAlerts(t, app.URL, geojsURL, clk.Now,
map[string]string{
trustedProxies: trustLocalhost,
alertWebhookURL: "https://alerts.example/smallwebwaf",
rateLimitPerMinute: "1",
})
s := &sender{t: t, addr: addr, out: out}
// The answer is kept before the requests, so GeoJS is not asked, and
// gives no answer of its own.
netblock := netip.MustParsePrefix(fromDE + "/32")
server.GeoJS.Load([]lookup.Answer{{
Client: netblock, ASN: asnDE, ASName: asNameDE, Country: "DE",
Answered: clk.Now(), Used: clk.Now(),
}})
want := asnAndCountry{asnDE, asNameDE, "DE"}
for _, line := range []logLine{
s.get(fromDE, http.StatusOK, requestlog.ActionForward),
s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited),
} {
if got := (asnAndCountry{line.ASN, line.ASName, line.Country}); got != want {
t.Errorf("log line has %+v, want %+v", got, want)
}
}
h := historyOf(t, server, fromDE)
if got := (asnAndCountry{h.ASN, h.ASName, h.Country}); got != want ||
!h.LookedUp.Equal(clk.Now()) {
t.Errorf("history has %+v, looked up at %s; want %+v, at %s",
got, h.LookedUp, want, clk.Now())
}
notes := server.Ledger.Bans(netblock)[0].Notes
if got := (asnAndCountry{notes.ASN, notes.ASName, notes.Country}); got != want {
t.Errorf("the ban's notes have %+v, want %+v", got, want)
}
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
if len(waiting) != 1 {
t.Fatalf("alerts waiting %+v, want the ban's alone", waiting)
}
alert := waiting[0]
if got := (asnAndCountry{alert.ASN, alert.ASName, alert.Country}); got != want {
t.Errorf("the ban's alert has %+v, want %+v", got, want)
}
}
func TestLookupSourceOffLooksNoClientUp(t *testing.T) {
t.Parallel()
geojsURL, asked := startGeoJS(t)
s, clk, server := startWithClock(t, geojsURL, map[string]string{lookupSource: "off"})
// Even an answer kept from before is not used.
server.GeoJS.Load([]lookup.Answer{{
Client: netip.MustParsePrefix(fromDE + "/32"), ASN: asnDE, ASName: asNameDE,
Country: "DE", Answered: clk.Now(), Used: clk.Now(),
}})
for _, from := range []string{fromDE, fromKP} {
line := s.get(from, http.StatusOK, requestlog.ActionForward)
got := asnAndCountry{line.ASN, line.ASName, line.Country}
if got != (asnAndCountry{}) {
t.Errorf("log line has %+v, want nothing", got)
}
}
if h := historyOf(t, server, fromDE); h.ASN != "" || !h.LookedUp.IsZero() {
t.Errorf("history has %q, looked up at %s, want no lookup", h.ASN, h.LookedUp)
}
if len(asked()) != 0 {
t.Errorf("GeoJS was asked about %v, want nothing", asked())
}
}
func TestLookupHeadersArePassedToTheAppAndTheClientsOwnRemoved(t *testing.T) {
t.Parallel()
var (
mu sync.Mutex
got [][2][]string // each request's X-Client-ASN and X-Client-Country
)
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
got = append(got, [2][]string{
r.Header.Values("X-Client-Asn"), r.Header.Values("X-Client-Country"),
})
})
geojsURL, _ := startGeoJS(t)
addr, out, _ := startProxyWithClock(t, app.URL, geojsURL, time.Now, map[string]string{
trustedProxies: trustLocalhost,
addLookupHeaders: "true",
})
s := &sender{t: t, addr: addr, out: out}
// Each client sends headers of its own. fromDE's first request waits
// for its answer, which the app is passed; unplaced has none to pass,
// and a client on a private address is not looked up.
own := "X-Client-ASN: AS1\r\nX-Client-Country: KP"
for _, from := range []string{fromDE, unplaced, "10.0.0.8"} {
s.requestWithHeader(from, "/", own, http.StatusOK, requestlog.ActionForward)
}
mu.Lock()
defer mu.Unlock()
want := [][2][]string{{{asnDE}, {"DE"}}, {nil, nil}, {nil, nil}}
if !slices.EqualFunc(got, want, func(a, b [2][]string) bool {
return slices.Equal(a[0], b[0]) && slices.Equal(a[1], b[1])
}) {
t.Errorf("the app was passed %v, want %v", got, want)
}
}
// waitUntil waits until done reports true, for at most waitLimit.
func waitUntil(done func() bool) {
deadline := time.Now().Add(waitLimit)
for !done() && time.Now().Before(deadline) {
time.Sleep(pollInterval)
}
}
+42 -1
View File
@@ -291,7 +291,8 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
metricsTopN: "2",
deniedCountries: "kp",
}
addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, env)
geojsURL, _ := startGeoJS(t)
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env)
// The answers are kept before the requests, so that none waits for
// GeoJS.
@@ -361,6 +362,46 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`)
}
func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
t.Parallel()
geojsURL, _ := startGeoJS(t)
s, clk, server := startWithClock(t, geojsURL, map[string]string{
metricsToken: token,
metricsTopN: "1",
})
// The answers are kept before the requests, so that GeoJS gives none
// of its own. Each client is in an AS of its own.
answer := func(addr, asn string) lookup.Answer {
return lookup.Answer{
Client: netip.MustParsePrefix(addr + "/32"), ASN: asn,
Answered: clk.Now(), Used: clk.Now(),
}
}
server.GeoJS.Load([]lookup.Answer{
answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
})
// With one AS number of its own, the other is counted as other. The
// metrics are asked for from a private address, which has no AS number.
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
s.get(fromKP, http.StatusOK, requestlog.ActionForward)
metrics := s.scrape("10.0.0.9")
wantMetric(t, metrics,
`smallwebwaf_asn_requests_total{asn="AS64501",instance="app"}`, 2)
wantMetric(t, metrics,
`smallwebwaf_asn_requests_total{asn="other",instance="app"}`, 1)
wantMetric(t, metrics,
`smallwebwaf_asn_request_bytes_total{asn="AS64501",instance="app"}`, 0)
wantMetric(t, metrics,
`smallwebwaf_asn_response_bytes_total{asn="other",instance="app"}`, 0)
wantNoSeries(t, metrics,
`smallwebwaf_asn_requests_total{asn="AS64502",instance="app"}`)
}
func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
t.Parallel()
+16 -9
View File
@@ -54,8 +54,8 @@ type Params struct {
RequestLog io.Writer
// ProcessLog receives the process's own messages.
ProcessLog *slog.Logger
// GeoJSURL is where clients' countries are looked up, normally
// lookup.URL. GeoJS is asked only while a country list is set.
// GeoJSURL is where clients' AS numbers and countries are looked up,
// normally lookup.URL, unless SWWAF_LOOKUP_SOURCE is off.
GeoJSURL string
// Now tells the time by which requests are counted for the rate
// limits, bans are made and run out, and GeoJS's answers are kept,
@@ -109,16 +109,23 @@ func New(params Params) *Server {
AttackBanDuration: params.Config.AttackBanDuration,
MaxBans: params.Config.MaxBans,
}),
geojs: lookup.New(lookup.Params{
URL: params.GeoJSURL,
Now: params.Now,
ProcessLog: params.ProcessLog,
Metrics: m,
Alerts: params.Alerts,
}),
rules: params.Rules,
alerts: params.Alerts,
}
h.geojs = lookup.New(lookup.Params{
URL: params.GeoJSURL,
Timeout: params.Config.LookupTimeout,
// The country lists and the headers act on the answer before the
// request goes on.
Wait: len(params.Config.DeniedCountries) > 0 ||
len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders,
Answered: h.addLookup,
Now: params.Now,
ProcessLog: params.ProcessLog,
Metrics: m,
Alerts: params.Alerts,
})
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
m.AddRules(params.Rules)
+12 -3
View File
@@ -66,6 +66,9 @@ const (
rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE"
rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY"
rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
lookupSource = "SWWAF_LOOKUP_SOURCE"
lookupTimeout = "SWWAF_LOOKUP_TIMEOUT"
addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS"
deniedCountries = "SWWAF_DENIED_COUNTRIES"
allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
banResponse = "SWWAF_BAN_RESPONSE"
@@ -203,8 +206,8 @@ func startProxy(t *testing.T, appURL string, env map[string]string) (string, *ou
return startProxyWithGeoJS(t, appURL, "", env)
}
// startProxyWithGeoJS is startProxy with clients' countries looked up at
// geojsURL.
// startProxyWithGeoJS is startProxy with clients' AS numbers and
// countries looked up at geojsURL.
func startProxyWithGeoJS(
t *testing.T, appURL, geojsURL string, env map[string]string,
) (string, *output) {
@@ -233,7 +236,9 @@ func startProxyWithClock(
// startProxyWithAlerts is startProxyWithClock, and returns the queue of
// the alerts the proxy raises as well, as the settings in env make it. No
// alert is sent from it: they wait in it, for the test to look at.
// alert is sent from it: they wait in it, for the test to look at. With
// no geojsURL, there is no stand-in for GeoJS to look clients up at, and
// SWWAF_LOOKUP_SOURCE is off unless env sets it.
func startProxyWithAlerts(
t *testing.T, appURL, geojsURL string, now func() time.Time,
env map[string]string,
@@ -243,6 +248,10 @@ func startProxyWithAlerts(
settings := map[string]string{
"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir(), instanceName: "app",
}
if geojsURL == "" {
settings[lookupSource] = "off"
}
maps.Copy(settings, env)
cfg, err := config.FromEnvironment(func(name string) (string, bool) {
+2 -1
View File
@@ -77,7 +77,8 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
const denied = "192.0.2.50" // in SWWAF_DENY_NETS
s, _, server := startWithClock(t, "", map[string]string{
geojsURL, _ := startGeoJS(t)
s, _, server := startWithClock(t, geojsURL, map[string]string{
rateLimitPerMinute: "1",
rateLimitExemptPaths: "/assets/,/favicon.ico",
denyNets: denied,
+37 -14
View File
@@ -48,7 +48,10 @@ type request struct {
client netip.Addr
peer netip.Addr
peerTrusted bool
start time.Time
// lookedUp is true once the client's AS number and country have been
// looked up, whether or not an answer was there.
lookedUp bool
start time.Time
// checked is when the checks were done, and upstreamStart when the
// request was handed to the app.
checked time.Time
@@ -192,14 +195,15 @@ func (rq *request) check(ctx context.Context) *refusal {
// checkClient runs the checks on the request's client, and returns the
// action of the first that refuses the request, or "" when none does. A
// client in SWWAF_ALLOW_NETS skips them. For any other client,
// SWWAF_DENY_NETS comes first, then a ban on its netblock, so that a
// client either refuses is not looked up, and then the country lists; a
// request any of them refuses is not counted for the rate limits. Then
// come the rate limits, unless the client is in
// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is exempt under
// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted,
// and last the rule files. ctx is the request's own context.
// client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any
// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock,
// so that a client either refuses is not looked up, then the lookup of
// its AS number and country, and then the country lists; a request any of
// them refuses is not counted for the rate limits. Then come the rate
// limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
// every other request is counted, and last the rule files. ctx is the
// request's own context.
func (rq *request) checkClient(ctx context.Context) string {
cfg := rq.h.config
if isInside(rq.client, cfg.AllowNets) {
@@ -216,7 +220,9 @@ func (rq *request) checkClient(ctx context.Context) string {
return requestlog.ActionBanned
}
if rq.countryDenied(ctx) {
rq.lookUp(ctx)
if rq.countryDenied() {
return requestlog.ActionCountryDenied
}
@@ -287,7 +293,8 @@ func (rq *request) forward(ctx context.Context) {
// rewrite makes the request the app receives: the client's request,
// unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers and
// the request's id set.
// the request's id set, and, while SWWAF_ADD_LOOKUP_HEADERS is set, the
// client's AS number and country.
func (rq *request) rewrite(pr *httputil.ProxyRequest) {
upstream := rq.h.config.UpstreamURL
pr.Out.URL.Scheme = upstream.Scheme
@@ -297,6 +304,10 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) {
pr.Out.URL.RawQuery = pr.In.URL.RawQuery
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
pr.Out.Header.Set(requestIDHeader, rq.line.RequestID)
if rq.h.config.AddLookupHeaders {
setLookupHeaders(pr.Out.Header, rq.line.ASN, rq.line.Country)
}
}
// modifyResponse looks at the app's answer before ReverseProxy passes it
@@ -474,7 +485,10 @@ func timing(start, end time.Time) *float64 {
}
// addToHistory adds the request, which has ended, to its client's
// history.
// history, and then, for a client that was looked up, the answer kept
// about it to that history and to the notes of the bans on its netblock:
// an answer that came during the request may have come before either was
// there, and one that comes later is added when it comes.
func (rq *request) addToHistory() {
var requestBytes int64
if rq.body != nil {
@@ -482,9 +496,9 @@ func (rq *request) addToHistory() {
}
forwarded := !rq.upstreamStart.IsZero()
group := clientGroup(rq.client)
rq.h.limiter.AddToHistory(clientGroup(rq.client), rq.h.now(), ratelimit.Request{
Country: rq.line.Country,
rq.h.limiter.AddToHistory(group, rq.h.now(), ratelimit.Request{
Forwarded: forwarded,
Refused: !forwarded && rq.refused.Load() != nil,
Status: rq.out.status,
@@ -492,6 +506,15 @@ func (rq *request) addToHistory() {
ResponseBytes: rq.out.bytes,
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
})
if !rq.lookedUp {
return
}
answer, kept := rq.h.geojs.Kept(group)
if kept {
rq.h.addLookup(answer)
}
}
// clientRequestDeadline is when the client must have sent its whole
+39 -4
View File
@@ -16,10 +16,10 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
start := midnight()
for i, r := range []ratelimit.Request{
{Country: "DE", Forwarded: true, Status: 200, RequestBytes: 10, ResponseBytes: 100},
{Forwarded: true, Status: 200, RequestBytes: 10, ResponseBytes: 100},
{Forwarded: true, Status: 101},
{Forwarded: true, Status: 304, RequestBytes: 5},
{Country: "FR", Refused: true, Status: 403, ResponseBytes: 10, BrokeLimit: true},
{Refused: true, Status: 403, ResponseBytes: 10, BrokeLimit: true},
{Forwarded: true, Status: 502, ResponseBytes: 12},
// Closed without an answer: refused, and no response.
{Refused: true, Status: 0},
@@ -33,8 +33,6 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
want := ratelimit.History{
FirstSeen: start,
LastSeen: start.Add(6 * time.Minute),
Country: "FR",
LookedUp: start.Add(3 * time.Minute),
Requests: 7,
Forwarded: 4,
Refused: 2,
@@ -52,6 +50,43 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
}
}
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{})
client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32")
start := midnight()
limiter.AddToHistory(client, start, ratelimit.Request{Forwarded: true})
limiter.AddLookup(client, start, "AS64496", "Example Net", "DE")
// A later answer replaces it, and one for a client the table does not
// hold adds no client.
limiter.AddLookup(client, start.Add(time.Hour), "AS64497", "Other Net", "FR")
limiter.AddLookup(other, start, "AS64496", "Example Net", "DE")
want := ratelimit.History{
FirstSeen: start,
LastSeen: start,
ASN: "AS64497",
ASName: "Other Net",
Country: "FR",
LookedUp: start.Add(time.Hour),
Requests: 1,
Forwarded: 1,
}
got := historyOf(t, limiter, client)
if got != want {
t.Errorf("history\n%+v\nwant\n%+v", got, want)
}
if clients := limiter.Snapshot(); len(clients) != 1 {
t.Errorf("the table holds %+v, want %s alone", clients, client)
}
}
func TestResetKeepsTheHistory(t *testing.T) {
t.Parallel()
+24 -9
View File
@@ -66,8 +66,12 @@ type Buckets struct {
type History struct {
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
// Country is the client's country as it was last looked up, and
// LookedUp when that was; both are empty while it never was.
// ASN, ASName and Country are the client's AS number, AS name and
// country as last looked up, each empty when the lookup could not
// find it, and LookedUp is when GeoJS gave that answer; all are empty
// while the client never was looked up.
ASN string `json:"asn,omitempty"`
ASName string `json:"as_name,omitempty"`
Country string `json:"country,omitempty"`
LookedUp time.Time `json:"looked_up,omitzero"`
// Requests are all the client's requests: Forwarded those passed to
@@ -103,8 +107,6 @@ type Offences struct {
// Request is what a client's history keeps of one of its requests.
type Request struct {
// Country is the client's country, when the request looked it up.
Country string
// Forwarded is true for a request passed to the app, Refused for one
// refused before anything reached it, a 401 at smallwebwaf's own
// endpoints included. Both are false for any other request smallwebwaf
@@ -209,11 +211,6 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) {
h.LastSeen = now
if r.Country != "" {
h.Country = r.Country
h.LookedUp = now
}
h.Requests++
if r.Forwarded {
h.Forwarded++
@@ -232,6 +229,24 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) {
}
}
// AddLookup gives client's history its AS number, AS name and country, as
// GeoJS gave them at lookedUp, if the table of clients holds the client.
// It does not make the client the most recently seen.
func (l *Limiter) AddLookup(
client netip.Prefix, lookedUp time.Time, asn, asName, country string,
) {
l.mu.Lock()
defer l.mu.Unlock()
c, held := l.clients.Peek(client)
if !held {
return
}
h := &c.History
h.ASN, h.ASName, h.Country, h.LookedUp = asn, asName, country, lookedUp
}
// Requests returns how many requests the clients inside netblock have
// sent, as their histories count them.
func (l *Limiter) Requests(netblock netip.Prefix) int64 {
+4 -1
View File
@@ -80,11 +80,14 @@ type Line struct {
// Request detail. RequestID is the X-Request-ID a trusted proxy sent,
// or a new one, and is sent on to the app. ForwardedFor is the
// X-Forwarded-For header as received. ClientGroup is the netblock the
// client is counted as.
// client is counted as. ASN, ASName and Country are the client's AS
// number, AS name and country, as looked up.
RequestID string `json:"request_id"`
PeerIP string `json:"peer_ip"`
ForwardedFor string `json:"forwarded_for,omitempty"`
ClientGroup string `json:"client_group"`
ASN string `json:"asn"`
ASName string `json:"as_name"`
Country string `json:"country"`
ContentType string `json:"content_type,omitempty"`
// ContentLength is the length of its body the request announced.
+5 -1
View File
@@ -104,12 +104,16 @@ func (o *output) text() string {
}
// run runs smallwebwaf with the settings in env until ctx is done, and
// returns its exit status.
// returns its exit status. SWWAF_LOOKUP_SOURCE is off unless env sets it,
// so that no test sends GeoJS its clients' addresses.
func run(ctx context.Context, env map[string]string, out *output) int {
return smallwebwaf.Run(ctx, smallwebwaf.Params{
Version: testVersion,
LookupEnv: func(name string) (string, bool) {
value, ok := env[name]
if !ok && name == "SWWAF_LOOKUP_SOURCE" {
return "off", true
}
return value, ok
},
+16 -3
View File
@@ -35,6 +35,9 @@ const (
clientsJSON = "clients.json"
lookupsJSON = "lookups.json"
alertsJSON = "alerts.json"
// The AS number and AS name the tests' clients are looked up in.
asn = "AS64496"
asName = "Example Net"
// What the process log says once Watch watches the directory, and as
// it takes in an edit.
watching = "watching the state files for edits"
@@ -55,6 +58,8 @@ const permanentBansJSON = `{
"cause": "admin",
"reason": "scrapes every commit",
"notes": {
"asn": "AS64496",
"as_name": "Example Net",
"country": "DE",
"limit": 1000,
"window": "minute",
@@ -1302,7 +1307,9 @@ func fill(params state.Params) {
client := netip.MustParsePrefix("203.0.113.9/32")
params.Ledger.Load([]bans.Ban{permanentBan()})
params.Ledger.BanForLimit(client, now, bans.Notes{Country: "DE", Limit: 1})
params.Ledger.BanForLimit(client, now, bans.Notes{
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
})
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Notes{RuleID: "env-file", Target: "path"})
@@ -1311,11 +1318,15 @@ func fill(params state.Params) {
}
params.Limiter.AddToHistory(client, now, ratelimit.Request{
Country: "DE", Forwarded: true, Status: 200, RequestBytes: 3, ResponseBytes: 5,
Forwarded: true, Status: 200, RequestBytes: 3, ResponseBytes: 5,
})
params.Limiter.AddLookup(client, now.Add(-time.Hour), asn, asName, "DE")
params.GeoJS.Load([]lookup.Answer{
{Client: client, Country: "DE", Answered: now.Add(-time.Hour), Used: now},
{
Client: client, ASN: asn, ASName: asName, Country: "DE",
Answered: now.Add(-time.Hour), Used: now,
},
{
Client: netip.MustParsePrefix("192.0.2.1/32"),
Answered: now.Add(-time.Hour), Used: now.Add(-time.Minute),
@@ -1350,6 +1361,8 @@ func permanentBan() bans.Ban {
Cause: bans.CauseAdmin,
Reason: "scrapes every commit",
Notes: bans.Notes{
ASN: asn,
ASName: asName,
Country: "DE",
Limit: 1000,
Window: "minute",