AS number and country looked up for every client (closes #95)
check / check (push) Waiting to run
check / check (push) Waiting to run
GeoJS's geo.json is asked about every new visitor unless SWWAF_LOOKUP_SOURCE is off. A request waits for its client's first answer only while a country list or SWWAF_ADD_LOOKUP_HEADERS needs it; otherwise the answer reaches the client's history and ban notes when it comes. The AS number and name go beside the country in the request log, history, ban notes, alerts and lookups.json, with metrics by AS number; 64512 counts as unknown. A client's own X-Client-ASN and X-Client-Country never reach the app, whatever the setting says, and make example-app sends no address to GeoJS. Judgement call: AS numbers are written AS64496, as SPEC's settings write them. Judgement call: SWWAF_LOOKUP_TIMEOUT is added, default 1s, and cannot be off. Model: opus-5-5
This commit is contained in:
@@ -137,8 +137,9 @@ type Params struct {
|
||||
}
|
||||
|
||||
// Alert is one alert, as the webhook is sent it and alerts.json holds it,
|
||||
// with the fields of the "Alert webhook schema" section of SPEC.md. ASN
|
||||
// and ASName are empty until AS numbers are looked up.
|
||||
// with the fields of the "Alert webhook schema" section of SPEC.md. ASN,
|
||||
// ASName and Country are, for a ban, the client's as the ban's notes give
|
||||
// them.
|
||||
//
|
||||
//nolint:tagliatelle // SPEC.md's alert webhook schema names its fields in snake_case
|
||||
type Alert struct {
|
||||
|
||||
+27
-1
@@ -91,7 +91,11 @@ func (b Ban) ActiveAt(now time.Time) bool {
|
||||
//
|
||||
//nolint:tagliatelle // the state files use snake_case, as the request log does
|
||||
type Notes struct {
|
||||
// Country is the client's country, when it was looked up.
|
||||
// ASN, ASName and Country are the client's AS number, AS name and
|
||||
// country, when they were looked up: when the request that caused the
|
||||
// ban was made, or when GeoJS answered about the client afterwards.
|
||||
ASN string `json:"asn"`
|
||||
ASName string `json:"as_name"`
|
||||
Country string `json:"country"`
|
||||
// Limit, Window and Count are, for a ban for a broken limit, the limit
|
||||
// that was broken, its window, "minute", "hour" or "day", and the
|
||||
@@ -411,6 +415,28 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban {
|
||||
return slices.Clone(*bans)
|
||||
}
|
||||
|
||||
// AddLookup gives the notes of netblock's bans that have no AS number, AS
|
||||
// name or country yet those of a client in it, as GeoJS answered about
|
||||
// it. It is not a request from netblock, and leaves when it was last seen
|
||||
// unchanged. It does not have bans.json written at once: the notes are
|
||||
// written with its next write, as the counts in them are.
|
||||
func (l *Ledger) AddLookup(netblock netip.Prefix, asn, asName, country string) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
bans, found := l.netblocks.Peek(netblock)
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
|
||||
for i := range *bans {
|
||||
notes := &(*bans)[i].Notes
|
||||
if notes.ASN == "" && notes.ASName == "" && notes.Country == "" {
|
||||
notes.ASN, notes.ASName, notes.Country = asn, asName, country
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Made returns how many bans for cause have been made since the start:
|
||||
// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an
|
||||
// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts
|
||||
|
||||
@@ -454,6 +454,45 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ledger := bans.New(defaultRules())
|
||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||
|
||||
// A ban made with the client's lookup, one made before it came, after
|
||||
// the first ended, and one on another netblock.
|
||||
ledger.BanForLimit(netblock, midnight(), bans.Notes{
|
||||
ASN: "AS64497", ASName: "Other Net", Country: "FR",
|
||||
})
|
||||
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
|
||||
ledger.BanForLimit(other, midnight(), bans.Notes{})
|
||||
|
||||
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
|
||||
|
||||
held := ledger.Bans(netblock)
|
||||
if len(held) != 2 {
|
||||
t.Fatalf("%s has %d bans, want 2", netblock, len(held))
|
||||
}
|
||||
|
||||
for i, want := range []bans.Notes{
|
||||
{ASN: "AS64497", ASName: "Other Net", Country: "FR"},
|
||||
{ASN: "AS64496", ASName: "Example Net", Country: "DE"},
|
||||
} {
|
||||
got := held[i].Notes
|
||||
if got.ASN != want.ASN || got.ASName != want.ASName || got.Country != want.Country {
|
||||
t.Errorf("ban %d's notes give %q, %q and %q, want %q, %q and %q", i+1,
|
||||
got.ASN, got.ASName, got.Country, want.ASN, want.ASName, want.Country)
|
||||
}
|
||||
}
|
||||
|
||||
if notes := ledger.Bans(other)[0].Notes; notes.ASN != "" || notes.Country != "" {
|
||||
t.Errorf("the ban on %s has %q and %q, want neither",
|
||||
other, notes.ASN, notes.Country)
|
||||
}
|
||||
}
|
||||
|
||||
// defaultRules are the rules at the settings' defaults.
|
||||
func defaultRules() bans.Rules {
|
||||
return bans.Rules{
|
||||
|
||||
@@ -91,6 +91,15 @@ type Config struct {
|
||||
// limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_PATHS).
|
||||
// Each starts with /.
|
||||
RateLimitExemptPaths []string
|
||||
// LookupSource is where each client's AS number and country are
|
||||
// looked up (SWWAF_LOOKUP_SOURCE): geojs, or off for nowhere. A request
|
||||
// waits up to LookupTimeout for its client's first answer while a
|
||||
// setting needs it (SWWAF_LOOKUP_TIMEOUT), which cannot be off.
|
||||
// AddLookupHeaders is true when the app is passed the client's AS
|
||||
// number and country in headers (SWWAF_ADD_LOOKUP_HEADERS).
|
||||
LookupSource string
|
||||
LookupTimeout time.Duration
|
||||
AddLookupHeaders bool
|
||||
// DeniedCountries are the countries whose clients are refused
|
||||
// (SWWAF_DENIED_COUNTRIES). ExclusivelyAllowedCountries, when not
|
||||
// empty, are the only countries whose clients are let through
|
||||
@@ -138,8 +147,8 @@ type Config struct {
|
||||
AdminToken string
|
||||
// MetricsToken is the bearer token a scraper sends for the metrics
|
||||
// (SWWAF_METRICS_TOKEN), "" while it is unset and the metrics are off.
|
||||
// MetricsTopN is how many countries get series of their own in the
|
||||
// metrics (SWWAF_METRICS_TOP_N).
|
||||
// MetricsTopN is how many AS numbers and how many countries get series
|
||||
// of their own in the metrics (SWWAF_METRICS_TOP_N).
|
||||
MetricsToken string
|
||||
MetricsTopN int
|
||||
// RulesDir is the directory of the rule files (SWWAF_RULES_DIR), read
|
||||
@@ -233,6 +242,8 @@ var (
|
||||
"is taken out of every request by Go's HTTP server, so it can never " +
|
||||
"be logged")
|
||||
errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too")
|
||||
errNotLookupSource = errors.New("is not geojs or off")
|
||||
errNeedsLookups = errors.New("it needs each client looked up")
|
||||
errNotOver4K = errors.New("is not a size of more than 4K, such as 32K")
|
||||
errNotDurationAboveZero = errors.New(
|
||||
"is not a duration above zero, such as 1h or 7d")
|
||||
@@ -300,6 +311,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
||||
RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"),
|
||||
RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"),
|
||||
RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""),
|
||||
LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"),
|
||||
LookupTimeout: env.durationNotOff("SWWAF_LOOKUP_TIMEOUT", "1s"),
|
||||
AddLookupHeaders: env.boolean("SWWAF_ADD_LOOKUP_HEADERS", "false"),
|
||||
DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""),
|
||||
ExclusivelyAllowedCountries: env.countries(
|
||||
"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
||||
@@ -339,13 +353,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
||||
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
||||
|
||||
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
||||
|
||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||
if slices.Contains(cfg.DeniedCountries, country) {
|
||||
env.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||
}
|
||||
}
|
||||
env.checkCountriesAndLookups(cfg)
|
||||
|
||||
if env.err != nil {
|
||||
return nil, env.err
|
||||
@@ -547,6 +555,47 @@ func (e *environment) countries(name, defaultValue string) []string {
|
||||
return countries
|
||||
}
|
||||
|
||||
// lookupSource reads the setting that is where clients are looked up:
|
||||
// geojs, or off.
|
||||
func (e *environment) lookupSource(name, defaultValue string) string {
|
||||
source := e.value(name, defaultValue)
|
||||
if source != "geojs" && source != off {
|
||||
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
||||
}
|
||||
|
||||
return source
|
||||
}
|
||||
|
||||
// checkCountriesAndLookups refuses a country on both country lists, and,
|
||||
// while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked
|
||||
// up: the country lists and SWWAF_ADD_LOOKUP_HEADERS.
|
||||
func (e *environment) checkCountriesAndLookups(cfg *Config) {
|
||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||
if slices.Contains(cfg.DeniedCountries, country) {
|
||||
e.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.LookupSource != off {
|
||||
return
|
||||
}
|
||||
|
||||
for _, setting := range []struct {
|
||||
name string
|
||||
set bool
|
||||
}{
|
||||
{"SWWAF_DENIED_COUNTRIES", len(cfg.DeniedCountries) > 0},
|
||||
{"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||
{"SWWAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
||||
} {
|
||||
if setting.set {
|
||||
e.check(setting.name, fmt.Errorf("is set while SWWAF_LOOKUP_SOURCE is off; %w",
|
||||
errNeedsLookups))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// headerNames reads a setting that is a list of header names, and
|
||||
// returns them in lower case.
|
||||
func (e *environment) headerNames(name, defaultValue string) []string {
|
||||
|
||||
@@ -40,6 +40,9 @@ const (
|
||||
rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR"
|
||||
rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY"
|
||||
rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
|
||||
lookupSource = "SWWAF_LOOKUP_SOURCE"
|
||||
lookupTimeout = "SWWAF_LOOKUP_TIMEOUT"
|
||||
addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS"
|
||||
deniedCountries = "SWWAF_DENIED_COUNTRIES"
|
||||
allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||
banResponse = "SWWAF_BAN_RESPONSE"
|
||||
@@ -117,6 +120,12 @@ const (
|
||||
// off switches a timeout, a size limit or a rate limit off.
|
||||
const off = "off"
|
||||
|
||||
// enabled is true, as a setting's value.
|
||||
const enabled = "true"
|
||||
|
||||
// defaultLookupSource is the default of SWWAF_LOOKUP_SOURCE.
|
||||
const defaultLookupSource = "geojs"
|
||||
|
||||
// environment is a set of environment variables, for FromEnvironment.
|
||||
type environment map[string]string
|
||||
|
||||
@@ -173,6 +182,9 @@ func TestDefaults(t *testing.T) {
|
||||
RulesDir: "/etc/smallwebwaf/rules.d",
|
||||
RulesEnabled: true,
|
||||
})
|
||||
wantLookupSettings(t, cfg, config.Config{
|
||||
LookupSource: defaultLookupSource, LookupTimeout: time.Second,
|
||||
})
|
||||
|
||||
if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" {
|
||||
t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL)
|
||||
@@ -773,6 +785,50 @@ func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupSettingsAsSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := fromEnvironment(t, environment{
|
||||
lookupTimeout: "500ms", addLookupHeaders: enabled,
|
||||
})
|
||||
wantLookupSettings(t, cfg, config.Config{
|
||||
LookupSource: defaultLookupSource, LookupTimeout: 500 * time.Millisecond,
|
||||
AddLookupHeaders: true,
|
||||
})
|
||||
|
||||
cfg = fromEnvironment(t, environment{lookupSource: off})
|
||||
wantLookupSettings(t, cfg, config.Config{
|
||||
LookupSource: off, LookupTimeout: time.Second,
|
||||
})
|
||||
}
|
||||
|
||||
func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, value := range map[string]string{
|
||||
deniedCountries: "kp",
|
||||
allowedCountries: "de",
|
||||
addLookupHeaders: enabled,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := config.FromEnvironment(environment{
|
||||
lookupSource: off, name: value,
|
||||
}.lookupEnv)
|
||||
if err == nil || !strings.HasPrefix(err.Error(), name+": ") ||
|
||||
!strings.Contains(err.Error(), lookupSource+" is off") {
|
||||
t.Errorf("error %v, want one naming %s and %s", err, name, lookupSource)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Set empty, the country lists need nothing looked up.
|
||||
fromEnvironment(t, environment{
|
||||
lookupSource: off, deniedCountries: "", allowedCountries: "",
|
||||
})
|
||||
}
|
||||
|
||||
func TestSizesAndOff(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -894,6 +950,9 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
|
||||
{rateLimitPerHour, "1.5"},
|
||||
{rateLimitPerDay, "-1"}, {rateLimitPerDay, "lots"},
|
||||
{rateLimitExemptPaths, "/assets/,,/static/"},
|
||||
{lookupSource, "file"}, {lookupSource, "GeoJS"}, {lookupSource, ""},
|
||||
{lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"},
|
||||
{addLookupHeaders, "yes"},
|
||||
{deniedCountries, "nk"},
|
||||
{deniedCountries, "kp,,ir"},
|
||||
{deniedCountries, "prk"},
|
||||
@@ -1181,6 +1240,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||
rateLimitPerHour: "10000",
|
||||
rateLimitPerDay: "50000",
|
||||
rateLimitExemptPaths: "",
|
||||
lookupSource: defaultLookupSource,
|
||||
lookupTimeout: "1s",
|
||||
addLookupHeaders: "false",
|
||||
deniedCountries: "",
|
||||
allowedCountries: "",
|
||||
banResponse: "403",
|
||||
@@ -1242,6 +1304,18 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) {
|
||||
wantBanSettings(t, got, want)
|
||||
}
|
||||
|
||||
// wantLookupSettings checks the settings for lookups.
|
||||
func wantLookupSettings(t *testing.T, got *config.Config, want config.Config) {
|
||||
t.Helper()
|
||||
|
||||
if got.LookupSource != want.LookupSource || got.LookupTimeout != want.LookupTimeout ||
|
||||
got.AddLookupHeaders != want.AddLookupHeaders {
|
||||
t.Errorf("lookups from %q, waited for %s, headers %t; want %q, %s, %t",
|
||||
got.LookupSource, got.LookupTimeout, got.AddLookupHeaders,
|
||||
want.LookupSource, want.LookupTimeout, want.AddLookupHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
// wantBanSettings checks the settings for bans, the state files, the
|
||||
// metrics and the rule files.
|
||||
func wantBanSettings(t *testing.T, got *config.Config, want config.Config) {
|
||||
|
||||
+124
-64
@@ -1,7 +1,7 @@
|
||||
// Package lookup looks up each client's country through the GeoJS web
|
||||
// service, and keeps the answers in memory, for at most 100,000 clients
|
||||
// and for 7 days each. The answers are written to lookups.json and read
|
||||
// from it by the state package.
|
||||
// Package lookup looks up each client's AS number and country through
|
||||
// the GeoJS web service, and keeps the answers in memory, for at most
|
||||
// 100,000 clients and for 7 days each. The answers are written to
|
||||
// lookups.json and read from it by the state package.
|
||||
package lookup
|
||||
|
||||
import (
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -23,9 +24,10 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
)
|
||||
|
||||
// URL is GeoJS's country endpoint. Asked about several addresses at once,
|
||||
// comma separated in its ip parameter, it answers with a list.
|
||||
const URL = "https://get.geojs.io/v1/ip/country.json"
|
||||
// URL is GeoJS's endpoint for an address's place and network. Asked about
|
||||
// several addresses at once, comma separated in its ip parameter, it
|
||||
// answers with a list.
|
||||
const URL = "https://get.geojs.io/v1/ip/geo.json"
|
||||
|
||||
const (
|
||||
// keepFor is how long an answer is used instead of asking GeoJS again.
|
||||
@@ -40,9 +42,8 @@ const (
|
||||
maxWaiting = 10000
|
||||
// maxPerRequest is how many addresses one request to GeoJS asks about.
|
||||
maxPerRequest = 200
|
||||
// timeout is how long a new client waits for its answer, and how long
|
||||
// a request to GeoJS may take before it is abandoned.
|
||||
timeout = time.Second
|
||||
// unknownASN is the AS number GeoJS gives when it knows none.
|
||||
unknownASN = 64512
|
||||
// After a failure GeoJS is not asked again for a second, and for
|
||||
// retryDelayFactor times as long after each further failure in a row,
|
||||
// up to five minutes.
|
||||
@@ -62,6 +63,16 @@ var (
|
||||
type Params struct {
|
||||
// URL is where GeoJS is asked, normally URL.
|
||||
URL string
|
||||
// Timeout is how long a request waits for its client's first answer,
|
||||
// and how long a request to GeoJS may take before it is abandoned
|
||||
// (SWWAF_LOOKUP_TIMEOUT).
|
||||
Timeout time.Duration
|
||||
// Wait is true when a setting needs each request's answer before the
|
||||
// request goes on. Otherwise no request waits for one.
|
||||
Wait bool
|
||||
// Answered, unless nil, is given each answer GeoJS gives, once it is
|
||||
// kept.
|
||||
Answered func(Answer)
|
||||
// Now tells the time, normally time.Now.
|
||||
Now func() time.Time
|
||||
// ProcessLog receives GeoJS's failures.
|
||||
@@ -73,11 +84,14 @@ type Params struct {
|
||||
Alerts *alerts.Queue
|
||||
}
|
||||
|
||||
// GeoJS looks up clients' countries through GeoJS. At most one request
|
||||
// to GeoJS is under way at a time, and it asks about every client waiting,
|
||||
// up to maxPerRequest. It is safe for concurrent use.
|
||||
// GeoJS looks up clients' AS numbers and countries through GeoJS. At most
|
||||
// one request to GeoJS is under way at a time, and it asks about every
|
||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||
type GeoJS struct {
|
||||
url string
|
||||
timeout time.Duration
|
||||
wait bool
|
||||
answered func(Answer)
|
||||
now func() time.Time
|
||||
processLog *slog.Logger
|
||||
metrics *metrics.Metrics
|
||||
@@ -100,10 +114,16 @@ type GeoJS struct {
|
||||
}
|
||||
|
||||
// Answer is what GeoJS said about a client, as lookups.json holds it: its
|
||||
// country, "" when GeoJS cannot place it, when GeoJS said so, and when
|
||||
// the answer was last used.
|
||||
// AS number, such as AS64496, and the AS's name, both "" when GeoJS knows
|
||||
// no AS number for it; its country, "" when GeoJS cannot place it; when
|
||||
// GeoJS said so, and when the answer was last used. The zero Answer is
|
||||
// that of a client with no answer.
|
||||
//
|
||||
//nolint:tagliatelle // the state files use snake_case, as the request log does
|
||||
type Answer struct {
|
||||
Client netip.Prefix `json:"client"`
|
||||
ASN string `json:"asn"`
|
||||
ASName string `json:"as_name"`
|
||||
Country string `json:"country"`
|
||||
Answered time.Time `json:"answered"`
|
||||
Used time.Time `json:"used"`
|
||||
@@ -128,6 +148,9 @@ func New(params Params) *GeoJS {
|
||||
|
||||
return &GeoJS{
|
||||
url: params.URL,
|
||||
timeout: params.Timeout,
|
||||
wait: params.Wait,
|
||||
answered: params.Answered,
|
||||
now: params.Now,
|
||||
processLog: params.ProcessLog,
|
||||
metrics: params.Metrics,
|
||||
@@ -142,23 +165,23 @@ func New(params Params) *GeoJS {
|
||||
}
|
||||
}
|
||||
|
||||
// Country returns the country GeoJS places client in, as a two-letter
|
||||
// code in capitals, or "" when the country cannot be found: GeoJS cannot
|
||||
// place the client, or has not answered in time. An answer is kept for 7
|
||||
// days. Without one, a client waits up to timeout for it, unless it has
|
||||
// gone without one before; until GeoJS answers, the client is asked about
|
||||
// again in the background. ctx is the context of the client's request,
|
||||
// and ends the wait when it ends.
|
||||
// LookUp returns the answer GeoJS gave about client, with its country as
|
||||
// a two-letter code in capitals, or the zero Answer when there is none
|
||||
// yet. An answer is kept for 7 days. Without one, the client is asked
|
||||
// about in the background, and, while Wait is set, the request waits up
|
||||
// to Timeout for the answer, unless the client has gone without one
|
||||
// before. ctx is the context of the client's request, and ends the wait
|
||||
// when it ends.
|
||||
//
|
||||
// GeoJS is asked about the client's first address, which is the client's
|
||||
// own address for IPv4, and an address in the same place for an IPv6 /64.
|
||||
func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
|
||||
country, asked := g.answerOrWait(ctx, client)
|
||||
func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer {
|
||||
answer, asked := g.answerOrWait(ctx, client)
|
||||
if asked == nil {
|
||||
return country
|
||||
return answer
|
||||
}
|
||||
|
||||
timer := time.NewTimer(timeout)
|
||||
timer := time.NewTimer(g.timeout)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
@@ -170,7 +193,7 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
country, found := g.kept(client)
|
||||
answer, found := g.kept(client)
|
||||
if !found {
|
||||
g.metrics.GeoJSUnanswered.Inc()
|
||||
}
|
||||
@@ -180,7 +203,15 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
|
||||
w.late = true
|
||||
}
|
||||
|
||||
return country
|
||||
return answer
|
||||
}
|
||||
|
||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
return g.kept(client)
|
||||
}
|
||||
|
||||
// Snapshot returns every answer kept, sorted by client, as lookups.json
|
||||
@@ -232,13 +263,13 @@ func (g *GeoJS) Load(answers []Answer) {
|
||||
// nil when there is nothing to wait for.
|
||||
func (g *GeoJS) answerOrWait(
|
||||
ctx context.Context, client netip.Prefix,
|
||||
) (string, <-chan struct{}) {
|
||||
) (Answer, <-chan struct{}) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
country, found := g.kept(client)
|
||||
answer, found := g.kept(client)
|
||||
if found {
|
||||
return country, nil
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
w, waiting := g.waiting[client]
|
||||
@@ -249,10 +280,14 @@ func (g *GeoJS) answerOrWait(
|
||||
|
||||
g.ask(ctx)
|
||||
|
||||
if !g.wait {
|
||||
return Answer{}, nil // the answer is not needed before the request goes on
|
||||
}
|
||||
|
||||
if w == nil {
|
||||
g.metrics.GeoJSUnanswered.Inc()
|
||||
|
||||
return "", nil // too many clients wait already
|
||||
return Answer{}, nil // too many clients wait already
|
||||
}
|
||||
|
||||
if !g.asking {
|
||||
@@ -263,25 +298,25 @@ func (g *GeoJS) answerOrWait(
|
||||
if w.late {
|
||||
g.metrics.GeoJSUnanswered.Inc()
|
||||
|
||||
return "", nil
|
||||
return Answer{}, nil
|
||||
}
|
||||
|
||||
return "", w.asked
|
||||
return Answer{}, w.asked
|
||||
}
|
||||
|
||||
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
||||
// and notes that it was used.
|
||||
func (g *GeoJS) kept(client netip.Prefix) (string, bool) {
|
||||
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||
now := g.now()
|
||||
|
||||
kept, found := g.answers.Get(client)
|
||||
if !found || now.Sub(kept.Answered) >= keepFor {
|
||||
return "", false
|
||||
return Answer{}, false
|
||||
}
|
||||
|
||||
kept.Used = now
|
||||
|
||||
return kept.Country, true
|
||||
return *kept, true
|
||||
}
|
||||
|
||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||
@@ -299,7 +334,8 @@ func (g *GeoJS) ask(ctx context.Context) {
|
||||
}
|
||||
|
||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||
// time, until none is left or GeoJS fails.
|
||||
// time, until none is left or GeoJS fails. Each answer kept is given to
|
||||
// Answered, outside the lock, since Answered takes locks of its own.
|
||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
for {
|
||||
clients := g.nextClients()
|
||||
@@ -307,8 +343,16 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
countries, err := g.request(ctx, clients)
|
||||
if !g.keep(clients, countries, err) {
|
||||
given, err := g.request(ctx, clients)
|
||||
kept, answered := g.keep(clients, given, err)
|
||||
|
||||
if g.answered != nil {
|
||||
for _, answer := range kept {
|
||||
g.answered(answer)
|
||||
}
|
||||
}
|
||||
|
||||
if !answered {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -340,32 +384,35 @@ func (g *GeoJS) nextClients() []netip.Prefix {
|
||||
return clients
|
||||
}
|
||||
|
||||
// keep notes how a request to GeoJS about clients ended, and reports
|
||||
// whether GeoJS answered about all of them. Each client whose address
|
||||
// GeoJS's answer names gets its answer, with no country when GeoJS gave
|
||||
// none. An answer that leaves an address out is a failure. After a
|
||||
// failure GeoJS is left alone for a while, and every client still waiting
|
||||
// stops waiting and is asked about once GeoJS is asked again.
|
||||
// keep notes how a request to GeoJS about clients ended, given being the
|
||||
// answer for each address GeoJS's answer names. It returns the answers it
|
||||
// kept, and reports whether GeoJS answered about all of the clients. Each
|
||||
// client whose address GeoJS's answer names gets its answer. An answer
|
||||
// that leaves an address out is a failure. After a failure GeoJS is left
|
||||
// alone for a while, and every client still waiting stops waiting and is
|
||||
// asked about once GeoJS is asked again.
|
||||
func (g *GeoJS) keep(
|
||||
clients []netip.Prefix, countries map[netip.Addr]string, err error,
|
||||
) bool {
|
||||
clients []netip.Prefix, given map[netip.Addr]Answer, err error,
|
||||
) ([]Answer, bool) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
now := g.now()
|
||||
kept := make([]Answer, 0, len(clients))
|
||||
leftOut := 0
|
||||
|
||||
for _, client := range clients {
|
||||
country, named := countries[client.Addr()]
|
||||
answer, named := given[client.Addr()]
|
||||
if !named {
|
||||
leftOut++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
g.answers.Add(client, &Answer{
|
||||
Client: client, Country: country, Answered: now, Used: now,
|
||||
})
|
||||
answer.Client, answer.Answered, answer.Used = client, now, now
|
||||
g.answers.Add(client, &answer)
|
||||
kept = append(kept, answer)
|
||||
|
||||
close(g.waiting[client].asked)
|
||||
delete(g.waiting, client)
|
||||
}
|
||||
@@ -400,26 +447,28 @@ func (g *GeoJS) keep(
|
||||
},
|
||||
})
|
||||
|
||||
return false
|
||||
return kept, false
|
||||
}
|
||||
|
||||
g.retryDelay = 0
|
||||
|
||||
return true
|
||||
return kept, true
|
||||
}
|
||||
|
||||
// request asks GeoJS about clients in one request, and returns the
|
||||
// country it gave, in capitals, for each address its answer names.
|
||||
// request asks GeoJS about clients in one request, and returns the answer
|
||||
// for each address GeoJS's answer names: its AS number and the AS's name,
|
||||
// both "" for the AS number 64512, which GeoJS gives when it knows none,
|
||||
// and its country, in capitals.
|
||||
func (g *GeoJS) request(
|
||||
ctx context.Context, clients []netip.Prefix,
|
||||
) (map[netip.Addr]string, error) {
|
||||
) (map[netip.Addr]Answer, error) {
|
||||
addrs := make([]string, 0, len(clients))
|
||||
|
||||
for _, client := range clients {
|
||||
addrs = append(addrs, client.Addr().String())
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
ctx, cancel := context.WithTimeout(ctx, g.timeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.url, http.NoBody)
|
||||
@@ -446,9 +495,12 @@ func (g *GeoJS) request(
|
||||
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
|
||||
}
|
||||
|
||||
//nolint:tagliatelle // GeoJS's own names
|
||||
var answers []struct {
|
||||
IP string `json:"ip"`
|
||||
Country string `json:"country"`
|
||||
IP string `json:"ip"`
|
||||
ASN int64 `json:"asn"`
|
||||
ASName string `json:"organization_name"`
|
||||
CountryCode string `json:"country_code"`
|
||||
}
|
||||
|
||||
err = json.NewDecoder(io.LimitReader(res.Body, maxResponseBytes)).Decode(&answers)
|
||||
@@ -456,14 +508,22 @@ func (g *GeoJS) request(
|
||||
return nil, fmt.Errorf("read GeoJS's answer: %w", err)
|
||||
}
|
||||
|
||||
countries := make(map[netip.Addr]string, len(answers))
|
||||
given := make(map[netip.Addr]Answer, len(answers))
|
||||
|
||||
for _, item := range answers {
|
||||
addr, err := netip.ParseAddr(item.IP)
|
||||
if err == nil {
|
||||
countries[addr] = strings.ToUpper(item.Country)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
answer := Answer{Country: strings.ToUpper(item.CountryCode)}
|
||||
if item.ASN != 0 && item.ASN != unknownASN {
|
||||
answer.ASN = "AS" + strconv.FormatInt(item.ASN, 10)
|
||||
answer.ASName = item.ASName
|
||||
}
|
||||
|
||||
given[addr] = answer
|
||||
}
|
||||
|
||||
return countries, nil
|
||||
return given, nil
|
||||
}
|
||||
|
||||
+183
-12
@@ -23,9 +23,13 @@ import (
|
||||
|
||||
const (
|
||||
// germany is where the stand-in for GeoJS places every address but
|
||||
// unplaced.
|
||||
germany = "DE"
|
||||
// unplaced is the address it cannot place.
|
||||
// unplaced, and asNumber, kept as asn, and asName the AS it gives them.
|
||||
germany = "DE"
|
||||
asNumber = 64496
|
||||
asn = "AS64496"
|
||||
asName = "Example Net"
|
||||
// unplaced is the address it cannot place, for which it gives the AS
|
||||
// number 64512 and the AS name Unknown, as GeoJS does.
|
||||
unplaced = "192.0.2.1"
|
||||
// leftOut is the address it leaves out of its answer when
|
||||
// answeringWithoutLeftOut.
|
||||
@@ -83,7 +87,7 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
|
||||
|
||||
var earlier sync.WaitGroup
|
||||
|
||||
earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) })
|
||||
earlier.Go(func() { g.LookUp(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) })
|
||||
defer earlier.Wait()
|
||||
|
||||
waitForRequests(t, geojs, 1)
|
||||
@@ -115,6 +119,58 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestWaitsAsLongAsTheTimeoutSaysAndGeoJSIsAbandonedAfterIt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
// A timeout longer than the default second, and a GeoJS that does
|
||||
// not answer.
|
||||
const longerTimeout = 3 * time.Second
|
||||
|
||||
m := metrics.New(1, "app")
|
||||
g := lookup.New(lookup.Params{
|
||||
URL: lookup.URL,
|
||||
Timeout: longerTimeout,
|
||||
Wait: true,
|
||||
Now: time.Now,
|
||||
ProcessLog: slog.New(slog.DiscardHandler),
|
||||
Metrics: m,
|
||||
Alerts: alerts.New(alerts.Params{}),
|
||||
})
|
||||
g.SetTransport(&standIn{answers: hanging})
|
||||
|
||||
var (
|
||||
request sync.WaitGroup
|
||||
waited time.Duration
|
||||
)
|
||||
|
||||
request.Go(func() {
|
||||
began := time.Now()
|
||||
|
||||
wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "")
|
||||
|
||||
waited = time.Since(began)
|
||||
})
|
||||
|
||||
// A moment before the timeout runs out, GeoJS is still being asked:
|
||||
// the request to it has not failed.
|
||||
time.Sleep(longerTimeout - time.Millisecond)
|
||||
synctest.Wait()
|
||||
wantFailures(t, m, 0)
|
||||
|
||||
// As it runs out, the client's request goes on, and the request to
|
||||
// GeoJS is abandoned, which counts as a failure.
|
||||
request.Wait()
|
||||
synctest.Wait()
|
||||
|
||||
if waited != longerTimeout {
|
||||
t.Errorf("waited %s for the answer, want %s", waited, longerTimeout)
|
||||
}
|
||||
|
||||
wantFailures(t, m, 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -187,6 +243,96 @@ func TestCountryIsKeptInCapitals(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestAnswerHoldsTheASNumberTheASNameAndTheCountry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
_, clock, g := start()
|
||||
placed := netip.MustParsePrefix("203.0.113.9/32")
|
||||
notPlaced := netip.MustParsePrefix(unplaced + "/32")
|
||||
now := clock.Now()
|
||||
|
||||
// For the client it cannot place, GeoJS gives the AS number 64512
|
||||
// and the AS name Unknown, which count as unknown.
|
||||
for client, want := range map[netip.Prefix]lookup.Answer{
|
||||
placed: {
|
||||
Client: placed, ASN: asn, ASName: asName, Country: germany,
|
||||
Answered: now, Used: now,
|
||||
},
|
||||
notPlaced: {Client: notPlaced, Answered: now, Used: now},
|
||||
} {
|
||||
got := g.LookUp(t.Context(), client)
|
||||
if got != want {
|
||||
t.Errorf("answer for %s\n%+v\nwant\n%+v", client, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithoutWaitTheRequestGoesOnAtOnceAndTheAnswerIsGivenWhenItComes(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
given []lookup.Answer
|
||||
)
|
||||
|
||||
geojs := &standIn{answers: answeringSlowly}
|
||||
clock := newClock()
|
||||
m := metrics.New(1, "app")
|
||||
g := lookup.New(lookup.Params{
|
||||
URL: lookup.URL,
|
||||
Timeout: timeout,
|
||||
Answered: func(answer lookup.Answer) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
given = append(given, answer)
|
||||
},
|
||||
Now: clock.Now,
|
||||
ProcessLog: slog.New(slog.DiscardHandler),
|
||||
Metrics: m,
|
||||
Alerts: alerts.New(alerts.Params{}),
|
||||
})
|
||||
g.SetTransport(geojs)
|
||||
|
||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||
|
||||
// The request goes on at once, without an answer, and GeoJS is asked
|
||||
// about the client, which it answers most of a second later.
|
||||
began := time.Now()
|
||||
|
||||
got := g.LookUp(t.Context(), client)
|
||||
if took := time.Since(began); took != 0 || got != (lookup.Answer{}) {
|
||||
t.Errorf("waited %s for %+v, want no wait and no answer", took, got)
|
||||
}
|
||||
|
||||
waitForRequests(t, geojs, 1)
|
||||
wantAsked(t, geojs, 0, "203.0.113.9")
|
||||
|
||||
time.Sleep(timeout)
|
||||
synctest.Wait()
|
||||
|
||||
now := clock.Now()
|
||||
want := lookup.Answer{
|
||||
Client: client, ASN: asn, ASName: asName, Country: germany,
|
||||
Answered: now, Used: now,
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
if !slices.Equal(given, []lookup.Answer{want}) {
|
||||
t.Errorf("answers given %+v, want only %+v", given, want)
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
wantCountry(t, g, client, germany)
|
||||
wantUnanswered(t, m, 0)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -197,6 +343,8 @@ func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) {
|
||||
geojs := &standIn{answers: hanging}
|
||||
g := lookup.New(lookup.Params{
|
||||
URL: lookup.URL,
|
||||
Timeout: timeout,
|
||||
Wait: true,
|
||||
Now: time.Now,
|
||||
ProcessLog: slog.New(slog.NewTextHandler(&log, nil)),
|
||||
Metrics: metrics.New(1, "app"),
|
||||
@@ -401,6 +549,8 @@ func TestClientsWithoutAnAnswerAreCounted(t *testing.T) {
|
||||
m := metrics.New(1, "app")
|
||||
g := lookup.New(lookup.Params{
|
||||
URL: lookup.URL,
|
||||
Timeout: timeout,
|
||||
Wait: true,
|
||||
Now: time.Now,
|
||||
ProcessLog: slog.New(slog.DiscardHandler),
|
||||
Metrics: m,
|
||||
@@ -494,21 +644,24 @@ func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
list := make([]map[string]string, 0, len(addrs))
|
||||
list := make([]map[string]any, 0, len(addrs))
|
||||
|
||||
for _, addr := range addrs {
|
||||
country := germany
|
||||
item := map[string]any{
|
||||
"ip": addr, "asn": asNumber, "organization_name": asName,
|
||||
"country_code": germany,
|
||||
}
|
||||
|
||||
switch {
|
||||
case addr == unplaced:
|
||||
country = ""
|
||||
item = map[string]any{"ip": addr, "asn": 64512, "organization_name": "Unknown"}
|
||||
case addr == leftOut && answers == answeringWithoutLeftOut:
|
||||
continue
|
||||
case answers == answeringInLowerCase:
|
||||
country = strings.ToLower(germany)
|
||||
item["country_code"] = strings.ToLower(germany)
|
||||
}
|
||||
|
||||
list = append(list, map[string]string{"ip": addr, "country": country})
|
||||
list = append(list, item)
|
||||
}
|
||||
|
||||
var answer any = list
|
||||
@@ -566,7 +719,8 @@ func (c *testClock) advance(d time.Duration) {
|
||||
}
|
||||
|
||||
// start returns a stand-in for GeoJS that answers, a clock, and a GeoJS
|
||||
// asking the stand-in by that clock.
|
||||
// asking the stand-in by that clock, for which a request waits for its
|
||||
// client's first answer.
|
||||
func start() (*standIn, *testClock, *lookup.GeoJS) {
|
||||
geojs, clock, g, _ := startWithAlerts()
|
||||
|
||||
@@ -578,7 +732,7 @@ func start() (*standIn, *testClock, *lookup.GeoJS) {
|
||||
// cooldown, by the same clock.
|
||||
func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) {
|
||||
geojs := &standIn{}
|
||||
clock := &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)}
|
||||
clock := newClock()
|
||||
queue := alerts.New(alerts.Params{
|
||||
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
||||
Events: alerts.Events(),
|
||||
@@ -587,6 +741,8 @@ func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) {
|
||||
})
|
||||
g := lookup.New(lookup.Params{
|
||||
URL: lookup.URL,
|
||||
Timeout: timeout,
|
||||
Wait: true,
|
||||
Now: clock.Now,
|
||||
ProcessLog: slog.New(slog.DiscardHandler),
|
||||
Metrics: metrics.New(1, "app"),
|
||||
@@ -597,6 +753,11 @@ func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) {
|
||||
return geojs, clock, g, queue
|
||||
}
|
||||
|
||||
// newClock returns a clock set to the start of a day.
|
||||
func newClock() *testClock {
|
||||
return &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)}
|
||||
}
|
||||
|
||||
// newClients returns what returns a new IPv4 client each time it is
|
||||
// called.
|
||||
func newClients() func() netip.Prefix {
|
||||
@@ -613,7 +774,7 @@ func newClients() func() netip.Prefix {
|
||||
func wantCountry(t *testing.T, g *lookup.GeoJS, client netip.Prefix, want string) {
|
||||
t.Helper()
|
||||
|
||||
got := g.Country(t.Context(), client)
|
||||
got := g.LookUp(t.Context(), client).Country
|
||||
if got != want {
|
||||
t.Errorf("%s is in %q, want %q", client, got, want)
|
||||
}
|
||||
@@ -658,6 +819,16 @@ func wantUnanswered(t *testing.T, m *metrics.Metrics, want float64) {
|
||||
}
|
||||
}
|
||||
|
||||
// wantFailures checks how many requests to GeoJS m counts as failed.
|
||||
func wantFailures(t *testing.T, m *metrics.Metrics, want float64) {
|
||||
t.Helper()
|
||||
|
||||
got := testutil.ToFloat64(m.GeoJSFailures)
|
||||
if got != want {
|
||||
t.Errorf("%v requests to GeoJS failed, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForRequests waits until g has done all it can before time passes,
|
||||
// checks that GeoJS has had count requests, and returns the addresses each
|
||||
// asked about.
|
||||
|
||||
@@ -26,8 +26,11 @@ func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
|
||||
wantCountry(t, g, placed, germany)
|
||||
|
||||
want := []lookup.Answer{
|
||||
{Client: notPlaced, Country: "", Answered: asked, Used: asked},
|
||||
{Client: placed, Country: germany, Answered: asked, Used: asked.Add(time.Hour)},
|
||||
{Client: notPlaced, Answered: asked, Used: asked},
|
||||
{
|
||||
Client: placed, ASN: asn, ASName: asName, Country: germany,
|
||||
Answered: asked, Used: asked.Add(time.Hour),
|
||||
},
|
||||
}
|
||||
if got := g.Snapshot(); !slices.Equal(got, want) {
|
||||
t.Errorf("snapshot\n%+v\nwant\n%+v", got, want)
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// other is the label value under which the countries or AS numbers
|
||||
// outside the busiest are counted.
|
||||
const other = "other"
|
||||
|
||||
// busiest are the metrics by one thing the lookup finds of the client,
|
||||
// its country or its AS number, for requests whose client's is known.
|
||||
// The topN busiest countries or AS numbers, by their requests since the
|
||||
// start, have series of their own, and the others are counted under
|
||||
// other, so that there are never more than topN + 1 series. One that
|
||||
// drops out of the busiest loses its series, and its next requests are
|
||||
// counted under other; one that becomes one of them gets a series that
|
||||
// counts from then on. Each series therefore only ever goes up.
|
||||
type busiest struct {
|
||||
topN int
|
||||
|
||||
requests *prometheus.CounterVec
|
||||
requestBytes *prometheus.CounterVec
|
||||
responseBytes *prometheus.CounterVec
|
||||
// refused are, by country, the requests the country lists refused; nil
|
||||
// by AS number.
|
||||
refused *prometheus.CounterVec
|
||||
|
||||
mu sync.Mutex
|
||||
// seen is each country's or AS number's requests since the start, by
|
||||
// which they are ranked.
|
||||
seen map[string]int64
|
||||
// top are those with series of their own.
|
||||
top map[string]bool
|
||||
}
|
||||
|
||||
// newCountries returns the metrics by country, with series of their own
|
||||
// for the topN busiest countries.
|
||||
func newCountries(topN int) *busiest {
|
||||
countries := newBusiest(topN, "country", "the client's country")
|
||||
countries.refused = counterVec("smallwebwaf_country_list_refusals_total",
|
||||
"Requests the country lists refused, by the client's country.",
|
||||
[]string{"country"})
|
||||
|
||||
return countries
|
||||
}
|
||||
|
||||
// newASNs returns the metrics by AS number, with series of their own for
|
||||
// the topN busiest AS numbers.
|
||||
func newASNs(topN int) *busiest {
|
||||
return newBusiest(topN, "asn", "the client's AS number")
|
||||
}
|
||||
|
||||
// newBusiest returns the metrics by label, which is described as
|
||||
// description, with series of their own for the topN busiest values.
|
||||
func newBusiest(topN int, label, description string) *busiest {
|
||||
by := []string{label}
|
||||
|
||||
return &busiest{
|
||||
topN: topN,
|
||||
requests: counterVec("smallwebwaf_"+label+"_requests_total",
|
||||
"Requests, by "+description+".", by),
|
||||
requestBytes: counterVec("smallwebwaf_"+label+"_request_bytes_total",
|
||||
"Request body bytes, by "+description+".", by),
|
||||
responseBytes: counterVec("smallwebwaf_"+label+"_response_bytes_total",
|
||||
"Response body bytes, by "+description+".", by),
|
||||
seen: map[string]int64{},
|
||||
top: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
// Describe and Collect make the metrics a prometheus.Collector, so that
|
||||
// they are registered together.
|
||||
func (b *busiest) Describe(ch chan<- *prometheus.Desc) {
|
||||
for _, vec := range b.vecs() {
|
||||
vec.Describe(ch)
|
||||
}
|
||||
}
|
||||
|
||||
// Collect is the other half of prometheus.Collector, with Describe.
|
||||
func (b *busiest) Collect(ch chan<- prometheus.Metric) {
|
||||
for _, vec := range b.vecs() {
|
||||
vec.Collect(ch)
|
||||
}
|
||||
}
|
||||
|
||||
// vecs returns the metrics: by AS number, those of requests and bytes; by
|
||||
// country, the refusals by the country lists as well.
|
||||
func (b *busiest) vecs() []*prometheus.CounterVec {
|
||||
vecs := []*prometheus.CounterVec{b.requests, b.requestBytes, b.responseBytes}
|
||||
if b.refused != nil {
|
||||
vecs = append(vecs, b.refused)
|
||||
}
|
||||
|
||||
return vecs
|
||||
}
|
||||
|
||||
// add counts a request from its log line, whose client's country or AS
|
||||
// number, value, is known.
|
||||
func (b *busiest) add(value string, line *requestlog.Line) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
b.seen[value]++
|
||||
|
||||
label := b.label(value)
|
||||
b.requests.WithLabelValues(label).Inc()
|
||||
b.requestBytes.WithLabelValues(label).Add(float64(line.RequestBytes))
|
||||
b.responseBytes.WithLabelValues(label).Add(float64(line.ResponseBytes))
|
||||
|
||||
if b.refused != nil && line.Action == requestlog.ActionCountryDenied {
|
||||
b.refused.WithLabelValues(label).Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// label returns the label a request from value is counted under: value
|
||||
// while it is one of the busiest, other while it is not. A value busier
|
||||
// than the least busy of them takes its place, and that one's series are
|
||||
// dropped.
|
||||
func (b *busiest) label(value string) string {
|
||||
if b.top[value] {
|
||||
return value
|
||||
}
|
||||
|
||||
if len(b.top) < b.topN {
|
||||
b.top[value] = true
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
least := ""
|
||||
|
||||
for top := range b.top {
|
||||
if least == "" || b.seen[top] < b.seen[least] {
|
||||
least = top
|
||||
}
|
||||
}
|
||||
|
||||
if b.seen[value] <= b.seen[least] {
|
||||
return other
|
||||
}
|
||||
|
||||
delete(b.top, least)
|
||||
|
||||
for _, vec := range b.vecs() {
|
||||
vec.DeleteLabelValues(least)
|
||||
}
|
||||
|
||||
b.top[value] = true
|
||||
|
||||
return value
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// other is the label under which the countries outside the busiest are
|
||||
// counted.
|
||||
const other = "other"
|
||||
|
||||
// countries are the metrics by the client's country, for requests whose
|
||||
// client's country is known. The topN busiest countries, by their requests
|
||||
// since the start, have series of their own, and the others are counted
|
||||
// under other, so that there are never more than topN + 1 series. A
|
||||
// country that drops out of the busiest loses its series, and its next
|
||||
// requests are counted under other; one that becomes one of them gets a
|
||||
// series that counts from then on. Each series therefore only ever goes
|
||||
// up.
|
||||
type countries struct {
|
||||
topN int
|
||||
|
||||
requests *prometheus.CounterVec
|
||||
requestBytes *prometheus.CounterVec
|
||||
responseBytes *prometheus.CounterVec
|
||||
// refused are the requests the country lists refused.
|
||||
refused *prometheus.CounterVec
|
||||
|
||||
mu sync.Mutex
|
||||
// seen is each country's requests since the start, by which the
|
||||
// countries are ranked. GeoJS gives two-letter codes, so it holds at
|
||||
// most a few hundred.
|
||||
seen map[string]int64
|
||||
// top are the countries with series of their own.
|
||||
top map[string]bool
|
||||
}
|
||||
|
||||
// newCountries returns the metrics by country, with series of their own
|
||||
// for the topN busiest countries.
|
||||
func newCountries(topN int) *countries {
|
||||
byCountry := []string{"country"}
|
||||
|
||||
return &countries{
|
||||
topN: topN,
|
||||
requests: counterVec("smallwebwaf_country_requests_total",
|
||||
"Requests, by the client's country.", byCountry),
|
||||
requestBytes: counterVec("smallwebwaf_country_request_bytes_total",
|
||||
"Request body bytes, by the client's country.", byCountry),
|
||||
responseBytes: counterVec("smallwebwaf_country_response_bytes_total",
|
||||
"Response body bytes, by the client's country.", byCountry),
|
||||
refused: counterVec("smallwebwaf_country_list_refusals_total",
|
||||
"Requests the country lists refused, by the client's country.",
|
||||
byCountry),
|
||||
seen: map[string]int64{},
|
||||
top: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
// add counts a request from its log line, whose country is known.
|
||||
func (c *countries) add(line *requestlog.Line) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.seen[line.Country]++
|
||||
|
||||
label := c.label(line.Country)
|
||||
c.requests.WithLabelValues(label).Inc()
|
||||
c.requestBytes.WithLabelValues(label).Add(float64(line.RequestBytes))
|
||||
c.responseBytes.WithLabelValues(label).Add(float64(line.ResponseBytes))
|
||||
|
||||
if line.Action == requestlog.ActionCountryDenied {
|
||||
c.refused.WithLabelValues(label).Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// label returns the label a request from country is counted under: the
|
||||
// country while it is one of the busiest, other while it is not. A
|
||||
// country busier than the least busy of them takes its place, and that
|
||||
// country's series are dropped.
|
||||
func (c *countries) label(country string) string {
|
||||
if c.top[country] {
|
||||
return country
|
||||
}
|
||||
|
||||
if len(c.top) < c.topN {
|
||||
c.top[country] = true
|
||||
|
||||
return country
|
||||
}
|
||||
|
||||
least := ""
|
||||
|
||||
for top := range c.top {
|
||||
if least == "" || c.seen[top] < c.seen[least] {
|
||||
least = top
|
||||
}
|
||||
}
|
||||
|
||||
if c.seen[country] <= c.seen[least] {
|
||||
return other
|
||||
}
|
||||
|
||||
delete(c.top, least)
|
||||
|
||||
for _, vec := range []*prometheus.CounterVec{
|
||||
c.requests, c.requestBytes, c.responseBytes, c.refused,
|
||||
} {
|
||||
vec.DeleteLabelValues(least)
|
||||
}
|
||||
|
||||
c.top[country] = true
|
||||
|
||||
return country
|
||||
}
|
||||
+16
-12
@@ -36,12 +36,13 @@ type Metrics struct {
|
||||
offences *prometheus.CounterVec
|
||||
// ruleMatches are made by AddRules.
|
||||
ruleMatches *prometheus.CounterVec
|
||||
countries *countries
|
||||
countries *busiest
|
||||
asns *busiest
|
||||
|
||||
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
|
||||
// that failed. GeoJSUnanswered are the requests whose client counted
|
||||
// as coming from an unknown country because GeoJS had not answered
|
||||
// about it in time.
|
||||
// that failed. GeoJSUnanswered are the requests that needed their
|
||||
// client's answer, for a setting that acts on it, and went on without
|
||||
// it because GeoJS had not given it in time.
|
||||
GeoJSRequests prometheus.Counter
|
||||
GeoJSFailures prometheus.Counter
|
||||
GeoJSUnanswered prometheus.Counter
|
||||
@@ -55,8 +56,8 @@ type Metrics struct {
|
||||
}
|
||||
|
||||
// New returns the metrics, with the Go runtime's and the process's own.
|
||||
// topN is how many countries get series of their own
|
||||
// (SWWAF_METRICS_TOP_N). Every metric carries instanceName
|
||||
// topN is how many countries and how many AS numbers get series of their
|
||||
// own (SWWAF_METRICS_TOP_N). Every metric carries instanceName
|
||||
// (SWWAF_INSTANCE_NAME) as its label instance.
|
||||
func New(topN int, instanceName string) *Metrics {
|
||||
byStatus := []string{"status_class", "action"}
|
||||
@@ -96,6 +97,7 @@ func New(topN int, instanceName string) *Metrics {
|
||||
offences: counterVec("smallwebwaf_offences_total",
|
||||
"Offences, by kind.", []string{"kind"}),
|
||||
countries: newCountries(topN),
|
||||
asns: newASNs(topN),
|
||||
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "smallwebwaf_geojs_requests_total",
|
||||
Help: "Requests to GeoJS.",
|
||||
@@ -106,8 +108,8 @@ func New(topN int, instanceName string) *Metrics {
|
||||
}),
|
||||
GeoJSUnanswered: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "smallwebwaf_geojs_unanswered_total",
|
||||
Help: "Requests whose client counted as coming from an unknown " +
|
||||
"country because GeoJS had not answered about it in time.",
|
||||
Help: "Requests that needed their client's answer from GeoJS and " +
|
||||
"went on without it, because GeoJS had not given it in time.",
|
||||
}),
|
||||
stateFileWrites: counterVec("smallwebwaf_state_file_writes_total",
|
||||
"Writes of each state file.", byFile),
|
||||
@@ -129,9 +131,7 @@ func New(topN int, instanceName string) *Metrics {
|
||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
||||
m.requestDuration, m.upstreamDuration,
|
||||
m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences,
|
||||
m.countries.requests, m.countries.requestBytes, m.countries.responseBytes,
|
||||
m.countries.refused,
|
||||
m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.countries, m.asns,
|
||||
m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered,
|
||||
m.stateFileWrites, m.stateFileWriteFailures,
|
||||
m.stateFileLastWrite, m.stateFileSize,
|
||||
@@ -315,7 +315,11 @@ func (m *Metrics) RequestEnded(
|
||||
}
|
||||
|
||||
if line.Country != "" {
|
||||
m.countries.add(line)
|
||||
m.countries.add(line.Country, line)
|
||||
}
|
||||
|
||||
if line.ASN != "" {
|
||||
m.asns.add(line.ASN, line)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -64,6 +64,8 @@ func (rq *request) limitBroken(now time.Time) bool {
|
||||
}
|
||||
|
||||
notes := bans.Notes{
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
Limit: hit.Limit,
|
||||
Window: hit.Window,
|
||||
@@ -103,6 +105,8 @@ func (rq *request) banForAttack(now time.Time, rule rules.Rule) {
|
||||
}
|
||||
|
||||
notes := bans.Notes{
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
RuleID: rule.ID,
|
||||
Target: rule.Target,
|
||||
@@ -165,6 +169,8 @@ func (rq *request) alertBan(ban bans.Ban) {
|
||||
Event: event,
|
||||
Client: rq.client,
|
||||
Netblock: ban.Netblock,
|
||||
ASN: ban.Notes.ASN,
|
||||
ASName: ban.Notes.ASName,
|
||||
Country: ban.Notes.Country,
|
||||
Reason: ban.Reason,
|
||||
Detail: detail,
|
||||
|
||||
@@ -281,6 +281,8 @@ func TestBanNotes(t *testing.T) {
|
||||
Cause: bans.CauseLimit,
|
||||
Reason: "requests per minute over the limit of 1",
|
||||
Notes: bans.Notes{
|
||||
ASN: asnDE,
|
||||
ASName: asNameDE,
|
||||
Country: "DE",
|
||||
Limit: 1,
|
||||
Window: minute,
|
||||
@@ -362,8 +364,9 @@ func (c *clock) advance(d time.Duration) {
|
||||
|
||||
// startWithClock starts smallwebwaf in front of an app that answers 200,
|
||||
// with the settings in env on top of trusting localhost's
|
||||
// X-Forwarded-For, clients' countries looked up at geojsURL, and a clock
|
||||
// set to midnight, the start of a bucket in every window.
|
||||
// X-Forwarded-For, clients' AS numbers and countries looked up at
|
||||
// geojsURL, and a clock set to midnight, the start of a bucket in every
|
||||
// window.
|
||||
func startWithClock(
|
||||
t *testing.T, geojsURL string, env map[string]string,
|
||||
) (*sender, *clock, *proxy.Server) {
|
||||
|
||||
@@ -1,31 +1,17 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"slices"
|
||||
)
|
||||
|
||||
// countryDenied reports whether the country lists refuse the request.
|
||||
// The client's country is looked up only while a list is set, and never
|
||||
// for a client on a private, loopback or link-local address, which has
|
||||
// no country. A client without a country, or whose country cannot be
|
||||
// found, is refused only by SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES. ctx is
|
||||
// the request's own context.
|
||||
func (rq *request) countryDenied(ctx context.Context) bool {
|
||||
// countryDenied reports whether the country lists refuse the request, by
|
||||
// the client's country as it was looked up. A client without a country,
|
||||
// or whose country cannot be found, is refused only by
|
||||
// SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
||||
func (rq *request) countryDenied() bool {
|
||||
denied := rq.h.config.DeniedCountries
|
||||
allowed := rq.h.config.ExclusivelyAllowedCountries
|
||||
|
||||
if len(denied) == 0 && len(allowed) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
var country string
|
||||
if hasCountry(rq.client) {
|
||||
country = rq.h.geojs.Country(ctx, clientGroup(rq.client))
|
||||
}
|
||||
|
||||
rq.line.Country = country
|
||||
country := rq.line.Country
|
||||
|
||||
if slices.Contains(denied, country) {
|
||||
return true
|
||||
@@ -33,9 +19,3 @@ func (rq *request) countryDenied(ctx context.Context) bool {
|
||||
|
||||
return len(allowed) > 0 && !slices.Contains(allowed, country)
|
||||
}
|
||||
|
||||
// hasCountry reports whether addr can be placed in a country: private,
|
||||
// loopback and link-local addresses cannot.
|
||||
func hasCountry(addr netip.Addr) bool {
|
||||
return !addr.IsPrivate() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast()
|
||||
}
|
||||
|
||||
@@ -56,16 +56,21 @@ func TestCountryLists(t *testing.T) {
|
||||
maps.Copy(env, tc.env)
|
||||
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env)
|
||||
|
||||
for i, sent := range []struct{ client, country string }{
|
||||
{fromDE, "DE"}, {fromKP, "KP"}, {unplaced, ""},
|
||||
// The AS number GeoJS gives unplaced, 64512, counts as unknown.
|
||||
for i, sent := range []struct{ client, asn, asName, country string }{
|
||||
{fromDE, asnDE, asNameDE, "DE"}, {fromKP, asnKP, asNameKP, "KP"},
|
||||
{unplaced, "", "", ""},
|
||||
} {
|
||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||
req.Header.Set(forwardedFor, sent.client)
|
||||
got := do(t, req)
|
||||
|
||||
line := out.requestLines(t, i+1)[i]
|
||||
if line.Country != sent.country {
|
||||
t.Errorf("log line has country %q, want %q", line.Country, sent.country)
|
||||
if line.ASN != sent.asn || line.ASName != sent.asName ||
|
||||
line.Country != sent.country {
|
||||
t.Errorf("log line has %q, %q and %q, want %q, %q and %q",
|
||||
line.ASN, line.ASName, line.Country,
|
||||
sent.asn, sent.asName, sent.country)
|
||||
}
|
||||
|
||||
if slices.Contains(tc.refused, sent.client) {
|
||||
@@ -130,7 +135,7 @@ func TestRequestRefusedByCountryIsNotCounted(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
answer := []map[string]string{{"ip": r.URL.Query().Get("ip"), "country": "DE"}}
|
||||
answer := []geojsAnswer{{IP: r.URL.Query().Get("ip"), CountryCode: "DE"}}
|
||||
|
||||
err := json.NewEncoder(w).Encode(answer)
|
||||
if err != nil {
|
||||
@@ -174,20 +179,15 @@ func TestRequestRefusedByCountryIsNotCounted(t *testing.T) {
|
||||
wantStatus(t, got, http.StatusOK)
|
||||
}
|
||||
|
||||
func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) {
|
||||
func TestPrivateAddressIsNeverLookedUp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
clients []string // "" sends no X-Forwarded-For: the client is 127.0.0.1
|
||||
name string
|
||||
env map[string]string
|
||||
}{
|
||||
{"no country list is set", nil, []string{fromKP, fromDE}},
|
||||
{
|
||||
"private, loopback and link-local addresses",
|
||||
map[string]string{deniedCountries: "kp"},
|
||||
[]string{"10.0.0.5", "192.168.1.9", "fd00::5", "", "169.254.0.9", "fe80::9"},
|
||||
},
|
||||
{"no setting needs the lookup", nil},
|
||||
{"a country list is set", map[string]string{deniedCountries: "kp"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -198,7 +198,10 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) {
|
||||
maps.Copy(env, tc.env)
|
||||
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env)
|
||||
|
||||
for i, sent := range tc.clients {
|
||||
// "" sends no X-Forwarded-For: the client is 127.0.0.1.
|
||||
for i, sent := range []string{
|
||||
"10.0.0.5", "192.168.1.9", "fd00::5", "", "169.254.0.9", "fe80::9",
|
||||
} {
|
||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||
if sent != "" {
|
||||
req.Header.Set(forwardedFor, sent)
|
||||
@@ -209,15 +212,26 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) {
|
||||
line := out.requestLines(t, i+1)[i]
|
||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
country, present := line.fields["country"]
|
||||
if !present || country != "" {
|
||||
t.Errorf("log line for %q has country %v, want an empty one",
|
||||
line.ClientIP, country)
|
||||
for _, field := range []string{"asn", "as_name", "country"} {
|
||||
value, present := line.fields[field]
|
||||
if !present || value != "" {
|
||||
t.Errorf("log line for %q has %s %v, want an empty one",
|
||||
line.ClientIP, field, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(asked()) != 0 {
|
||||
t.Errorf("GeoJS was asked about %v, want nothing", asked())
|
||||
// GeoJS is asked about up to 200 waiting clients at once, so once it
|
||||
// has been asked about fromDE, which comes last, it has been asked
|
||||
// about every client before it that waited for an answer.
|
||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||
req.Header.Set(forwardedFor, fromDE)
|
||||
wantStatus(t, do(t, req), http.StatusOK)
|
||||
|
||||
waitUntil(func() bool { return slices.Contains(asked(), fromDE) })
|
||||
|
||||
if got := asked(); !slices.Equal(got, []string{fromDE}) {
|
||||
t.Errorf("GeoJS was asked about %v, want %s alone", got, fromDE)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -264,18 +278,31 @@ func TestExclusiveListRefusesAPrivateAddressUnlessAllowed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP
|
||||
// and no other address. It returns its URL, and what returns the
|
||||
// addresses it has been asked about.
|
||||
// startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP,
|
||||
// each in an AS of its own, and no other address. It returns its URL, and
|
||||
// what returns the addresses it has been asked about.
|
||||
func startGeoJS(t *testing.T) (string, func() []string) {
|
||||
t.Helper()
|
||||
|
||||
places := map[string]string{fromDE: "DE", fromKP: "KP"}
|
||||
geojsURL, asked, release := startHeldGeoJS(t)
|
||||
release()
|
||||
|
||||
var asked struct {
|
||||
mu sync.Mutex
|
||||
addrs []string
|
||||
}
|
||||
return geojsURL, asked
|
||||
}
|
||||
|
||||
// startHeldGeoJS is startGeoJS for a stand-in that answers nothing until
|
||||
// release is called. Each request to it waits until then.
|
||||
func startHeldGeoJS(t *testing.T) (string, func() []string, func()) {
|
||||
t.Helper()
|
||||
|
||||
var (
|
||||
asked struct {
|
||||
mu sync.Mutex
|
||||
addrs []string
|
||||
}
|
||||
released = make(chan struct{})
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
geojs := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -285,11 +312,11 @@ func startGeoJS(t *testing.T) (string, func() []string) {
|
||||
asked.addrs = append(asked.addrs, addrs...)
|
||||
asked.mu.Unlock()
|
||||
|
||||
answers := make([]map[string]string, 0, len(addrs))
|
||||
<-released
|
||||
|
||||
answers := make([]geojsAnswer, 0, len(addrs))
|
||||
for _, addr := range addrs {
|
||||
answers = append(answers, map[string]string{
|
||||
"ip": addr, "country": places[addr],
|
||||
})
|
||||
answers = append(answers, answerAbout(addr))
|
||||
}
|
||||
|
||||
err := json.NewEncoder(w).Encode(answers)
|
||||
@@ -299,10 +326,48 @@ func startGeoJS(t *testing.T) (string, func() []string) {
|
||||
}))
|
||||
t.Cleanup(geojs.Close)
|
||||
|
||||
release := func() { once.Do(func() { close(released) }) }
|
||||
// Run before geojs.Close, which waits for every request to be answered.
|
||||
t.Cleanup(release)
|
||||
|
||||
return geojs.URL, func() []string {
|
||||
asked.mu.Lock()
|
||||
defer asked.mu.Unlock()
|
||||
|
||||
return slices.Clone(asked.addrs)
|
||||
}, release
|
||||
}
|
||||
|
||||
// The AS numbers and names the stand-in for GeoJS gives fromDE and
|
||||
// fromKP, as they are logged.
|
||||
const (
|
||||
asnDE = "AS64496"
|
||||
asNameDE = "Example Net"
|
||||
asnKP = "AS64511"
|
||||
asNameKP = "Other Net"
|
||||
)
|
||||
|
||||
// geojsAnswer is an answer of GeoJS about one address, with the fields
|
||||
// smallwebwaf reads.
|
||||
//
|
||||
//nolint:tagliatelle // GeoJS's own names
|
||||
type geojsAnswer struct {
|
||||
IP string `json:"ip"`
|
||||
ASN int `json:"asn"`
|
||||
ASName string `json:"organization_name"`
|
||||
CountryCode string `json:"country_code,omitempty"`
|
||||
}
|
||||
|
||||
// answerAbout is what the stand-in for GeoJS answers about addr: for an
|
||||
// address it cannot place, the AS number 64512 and the AS name Unknown
|
||||
// with no country, as GeoJS does.
|
||||
func answerAbout(addr string) geojsAnswer {
|
||||
switch addr {
|
||||
case fromDE:
|
||||
return geojsAnswer{IP: addr, ASN: 64496, ASName: asNameDE, CountryCode: "DE"}
|
||||
case fromKP:
|
||||
return geojsAnswer{IP: addr, ASN: 64511, ASName: asNameKP, CountryCode: "KP"}
|
||||
}
|
||||
|
||||
return geojsAnswer{IP: addr, ASN: 64512, ASName: "Unknown"}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,9 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
|
||||
start := clk.Now()
|
||||
|
||||
// Two let through, one over the limit, which bans the client, and one
|
||||
// refused under that ban, for which the country is not looked up.
|
||||
// refused under that ban, for which the client is not looked up. GeoJS
|
||||
// answers about the client at its first request, and its later ones
|
||||
// use that answer.
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
clk.advance(time.Second)
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
@@ -35,8 +37,10 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
|
||||
want := ratelimit.History{
|
||||
FirstSeen: start,
|
||||
LastSeen: start.Add(2 * time.Second),
|
||||
ASN: asnDE,
|
||||
ASName: asNameDE,
|
||||
Country: "DE",
|
||||
LookedUp: start.Add(time.Second),
|
||||
LookedUp: start,
|
||||
Requests: 4,
|
||||
Forwarded: 2,
|
||||
Refused: 2,
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||
)
|
||||
|
||||
// The headers in which the app is passed the client's AS number and
|
||||
// country while SWWAF_ADD_LOOKUP_HEADERS is set. Go writes every header
|
||||
// name in this form, as it sends it and as it receives it, so X-Client-ASN
|
||||
// arrives as X-Client-Asn, and Del removes a client's own whatever their
|
||||
// case; header names are not case-sensitive.
|
||||
const (
|
||||
asnHeader = "X-Client-Asn"
|
||||
countryHeader = "X-Client-Country"
|
||||
)
|
||||
|
||||
// lookUp looks up the client's AS number and country, and notes them for
|
||||
// the log line, unless SWWAF_LOOKUP_SOURCE is off or the client is on a
|
||||
// private, loopback or link-local address, which no lookup can place.
|
||||
// While a setting needs the answer, a new client's request waits for it.
|
||||
// ctx is the request's own context.
|
||||
func (rq *request) lookUp(ctx context.Context) {
|
||||
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
|
||||
return
|
||||
}
|
||||
|
||||
answer := rq.h.geojs.LookUp(ctx, clientGroup(rq.client))
|
||||
rq.lookedUp = true
|
||||
rq.line.ASN = answer.ASN
|
||||
rq.line.ASName = answer.ASName
|
||||
rq.line.Country = answer.Country
|
||||
}
|
||||
|
||||
// addLookup adds answer, GeoJS's answer about a client, to the client's
|
||||
// history, and to the notes of the bans on its netblock that have no AS
|
||||
// number, AS name or country yet.
|
||||
func (h *handler) addLookup(answer lookup.Answer) {
|
||||
h.limiter.AddLookup(answer.Client, answer.Answered,
|
||||
answer.ASN, answer.ASName, answer.Country)
|
||||
h.ledger.AddLookup(h.netblock(answer.Client.Addr()),
|
||||
answer.ASN, answer.ASName, answer.Country)
|
||||
}
|
||||
|
||||
// setLookupHeaders sets the headers in which the app is passed the
|
||||
// client's AS number and country, leaving out one that is unknown.
|
||||
func setLookupHeaders(header http.Header, asn, country string) {
|
||||
if asn != "" {
|
||||
header.Set(asnHeader, asn)
|
||||
}
|
||||
|
||||
if country != "" {
|
||||
header.Set(countryHeader, country)
|
||||
}
|
||||
}
|
||||
|
||||
// canBePlaced reports whether a lookup can place addr: private, loopback
|
||||
// and link-local addresses have no AS number or country.
|
||||
func canBePlaced(addr netip.Addr) bool {
|
||||
return !addr.IsPrivate() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast()
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
package proxy_test
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// asnAndCountry is what a lookup gives a client: its AS number, AS name
|
||||
// and country.
|
||||
type asnAndCountry struct{ asn, asName, country string }
|
||||
|
||||
func TestEveryClientIsLookedUpWithoutWaitingWhileNoSettingNeedsIt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The stand-in for GeoJS answers only once released. A request that
|
||||
// waited for it would wait an hour, and get no answer within
|
||||
// waitLimit.
|
||||
geojsURL, asked, release := startHeldGeoJS(t)
|
||||
s, _, server := startWithClock(t, geojsURL, map[string]string{
|
||||
lookupTimeout: "1h",
|
||||
rateLimitPerMinute: "1",
|
||||
})
|
||||
|
||||
// fromDE's second request breaks the limit and bans it, and fromKP
|
||||
// comes too. None waits for GeoJS.
|
||||
for _, line := range []logLine{
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward),
|
||||
s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited),
|
||||
s.get(fromKP, http.StatusOK, requestlog.ActionForward),
|
||||
} {
|
||||
got := asnAndCountry{line.ASN, line.ASName, line.Country}
|
||||
if got != (asnAndCountry{}) {
|
||||
t.Errorf("log line has %+v before GeoJS answered, want nothing", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Once GeoJS answers, each answer reaches the client's history, and
|
||||
// fromDE's reaches the notes of its ban.
|
||||
release()
|
||||
|
||||
netblock := netip.MustParsePrefix(fromDE + "/32")
|
||||
|
||||
waitUntil(func() bool {
|
||||
return historyOf(t, server, fromDE).ASN != "" &&
|
||||
historyOf(t, server, fromKP).ASN != "" &&
|
||||
server.Ledger.Bans(netblock)[0].Notes.ASN != ""
|
||||
})
|
||||
|
||||
de := asnAndCountry{asnDE, asNameDE, "DE"}
|
||||
|
||||
for addr, want := range map[string]asnAndCountry{
|
||||
fromDE: de, fromKP: {asnKP, asNameKP, "KP"},
|
||||
} {
|
||||
h := historyOf(t, server, addr)
|
||||
if got := (asnAndCountry{h.ASN, h.ASName, h.Country}); got != want {
|
||||
t.Errorf("%s's history has %+v, want %+v", addr, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
notes := server.Ledger.Bans(netblock)[0].Notes
|
||||
if got := (asnAndCountry{notes.ASN, notes.ASName, notes.Country}); got != de {
|
||||
t.Errorf("the ban's notes have %+v, want %+v", got, de)
|
||||
}
|
||||
|
||||
// GeoJS was asked about each client once, fromKP after fromDE, whose
|
||||
// request was under way when fromKP came.
|
||||
if got := asked(); !slices.Equal(got, []string{fromDE, fromKP}) {
|
||||
t.Errorf("GeoJS was asked about %v, want %s and %s", got, fromDE, fromKP)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||
clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)}
|
||||
addr, out, server, queue := startProxyWithAlerts(t, app.URL, geojsURL, clk.Now,
|
||||
map[string]string{
|
||||
trustedProxies: trustLocalhost,
|
||||
alertWebhookURL: "https://alerts.example/smallwebwaf",
|
||||
rateLimitPerMinute: "1",
|
||||
})
|
||||
s := &sender{t: t, addr: addr, out: out}
|
||||
|
||||
// The answer is kept before the requests, so GeoJS is not asked, and
|
||||
// gives no answer of its own.
|
||||
netblock := netip.MustParsePrefix(fromDE + "/32")
|
||||
server.GeoJS.Load([]lookup.Answer{{
|
||||
Client: netblock, ASN: asnDE, ASName: asNameDE, Country: "DE",
|
||||
Answered: clk.Now(), Used: clk.Now(),
|
||||
}})
|
||||
|
||||
want := asnAndCountry{asnDE, asNameDE, "DE"}
|
||||
|
||||
for _, line := range []logLine{
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward),
|
||||
s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited),
|
||||
} {
|
||||
if got := (asnAndCountry{line.ASN, line.ASName, line.Country}); got != want {
|
||||
t.Errorf("log line has %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
h := historyOf(t, server, fromDE)
|
||||
if got := (asnAndCountry{h.ASN, h.ASName, h.Country}); got != want ||
|
||||
!h.LookedUp.Equal(clk.Now()) {
|
||||
t.Errorf("history has %+v, looked up at %s; want %+v, at %s",
|
||||
got, h.LookedUp, want, clk.Now())
|
||||
}
|
||||
|
||||
notes := server.Ledger.Bans(netblock)[0].Notes
|
||||
if got := (asnAndCountry{notes.ASN, notes.ASName, notes.Country}); got != want {
|
||||
t.Errorf("the ban's notes have %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||
if len(waiting) != 1 {
|
||||
t.Fatalf("alerts waiting %+v, want the ban's alone", waiting)
|
||||
}
|
||||
|
||||
alert := waiting[0]
|
||||
if got := (asnAndCountry{alert.ASN, alert.ASName, alert.Country}); got != want {
|
||||
t.Errorf("the ban's alert has %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupSourceOffLooksNoClientUp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
geojsURL, asked := startGeoJS(t)
|
||||
s, clk, server := startWithClock(t, geojsURL, map[string]string{lookupSource: "off"})
|
||||
|
||||
// Even an answer kept from before is not used.
|
||||
server.GeoJS.Load([]lookup.Answer{{
|
||||
Client: netip.MustParsePrefix(fromDE + "/32"), ASN: asnDE, ASName: asNameDE,
|
||||
Country: "DE", Answered: clk.Now(), Used: clk.Now(),
|
||||
}})
|
||||
|
||||
for _, from := range []string{fromDE, fromKP} {
|
||||
line := s.get(from, http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
got := asnAndCountry{line.ASN, line.ASName, line.Country}
|
||||
if got != (asnAndCountry{}) {
|
||||
t.Errorf("log line has %+v, want nothing", got)
|
||||
}
|
||||
}
|
||||
|
||||
if h := historyOf(t, server, fromDE); h.ASN != "" || !h.LookedUp.IsZero() {
|
||||
t.Errorf("history has %q, looked up at %s, want no lookup", h.ASN, h.LookedUp)
|
||||
}
|
||||
|
||||
if len(asked()) != 0 {
|
||||
t.Errorf("GeoJS was asked about %v, want nothing", asked())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupHeadersArePassedToTheAppAndTheClientsOwnRemoved(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
got [][2][]string // each request's X-Client-ASN and X-Client-Country
|
||||
)
|
||||
|
||||
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
got = append(got, [2][]string{
|
||||
r.Header.Values("X-Client-Asn"), r.Header.Values("X-Client-Country"),
|
||||
})
|
||||
})
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
addr, out, _ := startProxyWithClock(t, app.URL, geojsURL, time.Now, map[string]string{
|
||||
trustedProxies: trustLocalhost,
|
||||
addLookupHeaders: "true",
|
||||
})
|
||||
s := &sender{t: t, addr: addr, out: out}
|
||||
|
||||
// Each client sends headers of its own. fromDE's first request waits
|
||||
// for its answer, which the app is passed; unplaced has none to pass,
|
||||
// and a client on a private address is not looked up.
|
||||
for _, from := range []string{fromDE, unplaced, "10.0.0.8"} {
|
||||
s.requestWithHeader(from, "/", clientsOwnLookupHeaders,
|
||||
http.StatusOK, requestlog.ActionForward)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
want := [][2][]string{{{asnDE}, {"DE"}}, {nil, nil}, {nil, nil}}
|
||||
if !slices.EqualFunc(got, want, func(a, b [2][]string) bool {
|
||||
return slices.Equal(a[0], b[0]) && slices.Equal(a[1], b[1])
|
||||
}) {
|
||||
t.Errorf("the app was passed %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientsOwnLookupHeadersAreRemovedWhileTheSettingIsOff(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
asn, country []string
|
||||
)
|
||||
|
||||
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
asn, country = r.Header.Values("X-Client-Asn"), r.Header.Values("X-Client-Country")
|
||||
})
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
addr, out, _ := startProxyWithClock(t, app.URL, geojsURL, time.Now, map[string]string{
|
||||
trustedProxies: trustLocalhost,
|
||||
})
|
||||
s := &sender{t: t, addr: addr, out: out}
|
||||
|
||||
s.requestWithHeader(fromDE, "/", clientsOwnLookupHeaders,
|
||||
http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
if asn != nil || country != nil {
|
||||
t.Errorf("the app was passed X-Client-ASN %v and X-Client-Country %v, want neither",
|
||||
asn, country)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestWaitsAsLongAsTheLookupTimeoutSays(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The test runs in a synctest bubble, where the time package runs on a
|
||||
// clock of the test's own: the wait lasts exactly as long as it should,
|
||||
// however slowly the test process runs. Nothing in it may wait on the
|
||||
// network, which would keep that clock from moving on: the request is
|
||||
// handed to the proxy's handler, and GeoJS is one that never answers.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
// Not the default second. The exclusive list needs the answer, and
|
||||
// the app is never reached.
|
||||
const timeout = 3 * time.Second
|
||||
|
||||
server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL,
|
||||
time.Now, map[string]string{
|
||||
lookupTimeout: timeout.String(),
|
||||
allowedCountries: "DE",
|
||||
})
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/",
|
||||
http.NoBody)
|
||||
req.RemoteAddr = net.JoinHostPort(fromDE, "1234")
|
||||
began := time.Now()
|
||||
|
||||
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||
|
||||
if waited := time.Since(began); waited != timeout {
|
||||
t.Errorf("the request waited %s for its answer, want %s", waited, timeout)
|
||||
}
|
||||
|
||||
// Without an answer, the client is in no country the list allows.
|
||||
wantLine(t, out.requestLine(t), http.StatusForbidden,
|
||||
requestlog.ActionCountryDenied)
|
||||
})
|
||||
}
|
||||
|
||||
// unansweredGeoJSURL is where a GeoJS that never answers is asked: a
|
||||
// request to it waits, without the network, until it is abandoned.
|
||||
// TestMain registers it with Go's default transport, through which GeoJS
|
||||
// is asked.
|
||||
const unansweredGeoJSURL = "unanswered://geojs/v1/ip/geo.json"
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
transport, _ := http.DefaultTransport.(*http.Transport)
|
||||
transport.RegisterProtocol("unanswered", unansweredGeoJS{})
|
||||
|
||||
m.Run()
|
||||
}
|
||||
|
||||
// unansweredGeoJS is the GeoJS at unansweredGeoJSURL.
|
||||
type unansweredGeoJS struct{}
|
||||
|
||||
// RoundTrip waits until req is abandoned.
|
||||
func (unansweredGeoJS) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
<-req.Context().Done()
|
||||
|
||||
return nil, req.Context().Err()
|
||||
}
|
||||
|
||||
// clientsOwnLookupHeaders are the X-Client-ASN and X-Client-Country a
|
||||
// client sends of its own, each twice, in two cases.
|
||||
const clientsOwnLookupHeaders = "X-Client-ASN: AS1\r\nx-client-asn: AS2\r\n" +
|
||||
"X-CLIENT-COUNTRY: KP\r\nx-client-country: CN"
|
||||
|
||||
// waitUntil waits until done reports true, for at most waitLimit.
|
||||
func waitUntil(done func() bool) {
|
||||
deadline := time.Now().Add(waitLimit)
|
||||
for !done() && time.Now().Before(deadline) {
|
||||
time.Sleep(pollInterval)
|
||||
}
|
||||
}
|
||||
@@ -291,7 +291,8 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
||||
metricsTopN: "2",
|
||||
deniedCountries: "kp",
|
||||
}
|
||||
addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, env)
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env)
|
||||
|
||||
// The answers are kept before the requests, so that none waits for
|
||||
// GeoJS.
|
||||
@@ -361,6 +362,46 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
||||
`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`)
|
||||
}
|
||||
|
||||
func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
s, clk, server := startWithClock(t, geojsURL, map[string]string{
|
||||
metricsToken: token,
|
||||
metricsTopN: "1",
|
||||
})
|
||||
|
||||
// The answers are kept before the requests, so that GeoJS gives none
|
||||
// of its own. Each client is in an AS of its own.
|
||||
answer := func(addr, asn string) lookup.Answer {
|
||||
return lookup.Answer{
|
||||
Client: netip.MustParsePrefix(addr + "/32"), ASN: asn,
|
||||
Answered: clk.Now(), Used: clk.Now(),
|
||||
}
|
||||
}
|
||||
server.GeoJS.Load([]lookup.Answer{
|
||||
answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
|
||||
})
|
||||
|
||||
// With one AS number of its own, the other is counted as other. The
|
||||
// metrics are asked for from a private address, which has no AS number.
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
s.get(fromKP, http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
metrics := s.scrape("10.0.0.9")
|
||||
wantMetric(t, metrics,
|
||||
`smallwebwaf_asn_requests_total{asn="AS64501",instance="app"}`, 2)
|
||||
wantMetric(t, metrics,
|
||||
`smallwebwaf_asn_requests_total{asn="other",instance="app"}`, 1)
|
||||
wantMetric(t, metrics,
|
||||
`smallwebwaf_asn_request_bytes_total{asn="AS64501",instance="app"}`, 0)
|
||||
wantMetric(t, metrics,
|
||||
`smallwebwaf_asn_response_bytes_total{asn="other",instance="app"}`, 0)
|
||||
wantNoSeries(t, metrics,
|
||||
`smallwebwaf_asn_requests_total{asn="AS64502",instance="app"}`)
|
||||
}
|
||||
|
||||
func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+16
-9
@@ -54,8 +54,8 @@ type Params struct {
|
||||
RequestLog io.Writer
|
||||
// ProcessLog receives the process's own messages.
|
||||
ProcessLog *slog.Logger
|
||||
// GeoJSURL is where clients' countries are looked up, normally
|
||||
// lookup.URL. GeoJS is asked only while a country list is set.
|
||||
// GeoJSURL is where clients' AS numbers and countries are looked up,
|
||||
// normally lookup.URL, unless SWWAF_LOOKUP_SOURCE is off.
|
||||
GeoJSURL string
|
||||
// Now tells the time by which requests are counted for the rate
|
||||
// limits, bans are made and run out, and GeoJS's answers are kept,
|
||||
@@ -109,16 +109,23 @@ func New(params Params) *Server {
|
||||
AttackBanDuration: params.Config.AttackBanDuration,
|
||||
MaxBans: params.Config.MaxBans,
|
||||
}),
|
||||
geojs: lookup.New(lookup.Params{
|
||||
URL: params.GeoJSURL,
|
||||
Now: params.Now,
|
||||
ProcessLog: params.ProcessLog,
|
||||
Metrics: m,
|
||||
Alerts: params.Alerts,
|
||||
}),
|
||||
rules: params.Rules,
|
||||
alerts: params.Alerts,
|
||||
}
|
||||
h.geojs = lookup.New(lookup.Params{
|
||||
URL: params.GeoJSURL,
|
||||
Timeout: params.Config.LookupTimeout,
|
||||
// The country lists and the headers act on the answer before the
|
||||
// request goes on.
|
||||
Wait: len(params.Config.DeniedCountries) > 0 ||
|
||||
len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
||||
params.Config.AddLookupHeaders,
|
||||
Answered: h.addLookup,
|
||||
Now: params.Now,
|
||||
ProcessLog: params.ProcessLog,
|
||||
Metrics: m,
|
||||
Alerts: params.Alerts,
|
||||
})
|
||||
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
|
||||
m.AddRules(params.Rules)
|
||||
|
||||
|
||||
@@ -66,6 +66,9 @@ const (
|
||||
rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE"
|
||||
rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY"
|
||||
rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
|
||||
lookupSource = "SWWAF_LOOKUP_SOURCE"
|
||||
lookupTimeout = "SWWAF_LOOKUP_TIMEOUT"
|
||||
addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS"
|
||||
deniedCountries = "SWWAF_DENIED_COUNTRIES"
|
||||
allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||
banResponse = "SWWAF_BAN_RESPONSE"
|
||||
@@ -203,8 +206,8 @@ func startProxy(t *testing.T, appURL string, env map[string]string) (string, *ou
|
||||
return startProxyWithGeoJS(t, appURL, "", env)
|
||||
}
|
||||
|
||||
// startProxyWithGeoJS is startProxy with clients' countries looked up at
|
||||
// geojsURL.
|
||||
// startProxyWithGeoJS is startProxy with clients' AS numbers and
|
||||
// countries looked up at geojsURL.
|
||||
func startProxyWithGeoJS(
|
||||
t *testing.T, appURL, geojsURL string, env map[string]string,
|
||||
) (string, *output) {
|
||||
@@ -232,17 +235,50 @@ func startProxyWithClock(
|
||||
}
|
||||
|
||||
// startProxyWithAlerts is startProxyWithClock, and returns the queue of
|
||||
// the alerts the proxy raises as well, as the settings in env make it. No
|
||||
// alert is sent from it: they wait in it, for the test to look at.
|
||||
// the alerts the proxy raises as well, as newProxy makes them.
|
||||
func startProxyWithAlerts(
|
||||
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||
env map[string]string,
|
||||
) (string, *output, *proxy.Server, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
server, out, alertQueue := newProxy(t, appURL, geojsURL, now, env)
|
||||
|
||||
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
_ = server.Serve(listener)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = server.Close()
|
||||
})
|
||||
|
||||
return listener.Addr().String(), out, server, alertQueue
|
||||
}
|
||||
|
||||
// newProxy makes the server startProxyWithClock starts, without starting
|
||||
// it, and returns it, what it writes, and the queue of the alerts the
|
||||
// proxy raises, as the settings in env make it. No alert is sent from the
|
||||
// queue: they wait in it, for the test to look at. With no geojsURL, there
|
||||
// is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE
|
||||
// is off unless env sets it.
|
||||
func newProxy(
|
||||
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||
env map[string]string,
|
||||
) (*proxy.Server, *output, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir(), instanceName: "app",
|
||||
}
|
||||
if geojsURL == "" {
|
||||
settings[lookupSource] = "off"
|
||||
}
|
||||
|
||||
maps.Copy(settings, env)
|
||||
|
||||
cfg, err := config.FromEnvironment(func(name string) (string, bool) {
|
||||
@@ -284,20 +320,7 @@ func startProxyWithAlerts(
|
||||
Alerts: alertQueue,
|
||||
})
|
||||
|
||||
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
_ = server.Serve(listener)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = server.Close()
|
||||
})
|
||||
|
||||
return listener.Addr().String(), out, server, alertQueue
|
||||
return server, out, alertQueue
|
||||
}
|
||||
|
||||
// newClient returns an HTTP client that sends requests as they are made,
|
||||
|
||||
@@ -77,7 +77,8 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
|
||||
|
||||
const denied = "192.0.2.50" // in SWWAF_DENY_NETS
|
||||
|
||||
s, _, server := startWithClock(t, "", map[string]string{
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
s, _, server := startWithClock(t, geojsURL, map[string]string{
|
||||
rateLimitPerMinute: "1",
|
||||
rateLimitExemptPaths: "/assets/,/favicon.ico",
|
||||
denyNets: denied,
|
||||
|
||||
+40
-14
@@ -48,7 +48,10 @@ type request struct {
|
||||
client netip.Addr
|
||||
peer netip.Addr
|
||||
peerTrusted bool
|
||||
start time.Time
|
||||
// lookedUp is true once the client's AS number and country have been
|
||||
// looked up, whether or not an answer was there.
|
||||
lookedUp bool
|
||||
start time.Time
|
||||
// checked is when the checks were done, and upstreamStart when the
|
||||
// request was handed to the app.
|
||||
checked time.Time
|
||||
@@ -192,14 +195,15 @@ func (rq *request) check(ctx context.Context) *refusal {
|
||||
|
||||
// checkClient runs the checks on the request's client, and returns the
|
||||
// action of the first that refuses the request, or "" when none does. A
|
||||
// client in SWWAF_ALLOW_NETS skips them. For any other client,
|
||||
// SWWAF_DENY_NETS comes first, then a ban on its netblock, so that a
|
||||
// client either refuses is not looked up, and then the country lists; a
|
||||
// request any of them refuses is not counted for the rate limits. Then
|
||||
// come the rate limits, unless the client is in
|
||||
// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is exempt under
|
||||
// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted,
|
||||
// and last the rule files. ctx is the request's own context.
|
||||
// client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any
|
||||
// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock,
|
||||
// so that a client either refuses is not looked up, then the lookup of
|
||||
// its AS number and country, and then the country lists; a request any of
|
||||
// them refuses is not counted for the rate limits. Then come the rate
|
||||
// limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
|
||||
// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
|
||||
// every other request is counted, and last the rule files. ctx is the
|
||||
// request's own context.
|
||||
func (rq *request) checkClient(ctx context.Context) string {
|
||||
cfg := rq.h.config
|
||||
if isInside(rq.client, cfg.AllowNets) {
|
||||
@@ -216,7 +220,9 @@ func (rq *request) checkClient(ctx context.Context) string {
|
||||
return requestlog.ActionBanned
|
||||
}
|
||||
|
||||
if rq.countryDenied(ctx) {
|
||||
rq.lookUp(ctx)
|
||||
|
||||
if rq.countryDenied() {
|
||||
return requestlog.ActionCountryDenied
|
||||
}
|
||||
|
||||
@@ -287,7 +293,9 @@ func (rq *request) forward(ctx context.Context) {
|
||||
|
||||
// rewrite makes the request the app receives: the client's request,
|
||||
// unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers and
|
||||
// the request's id set.
|
||||
// the request's id set, without any X-Client-ASN or X-Client-Country the
|
||||
// client sent, whatever SWWAF_ADD_LOOKUP_HEADERS says, and, while it is
|
||||
// set, with the client's AS number and country in them.
|
||||
func (rq *request) rewrite(pr *httputil.ProxyRequest) {
|
||||
upstream := rq.h.config.UpstreamURL
|
||||
pr.Out.URL.Scheme = upstream.Scheme
|
||||
@@ -297,6 +305,12 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) {
|
||||
pr.Out.URL.RawQuery = pr.In.URL.RawQuery
|
||||
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
|
||||
pr.Out.Header.Set(requestIDHeader, rq.line.RequestID)
|
||||
pr.Out.Header.Del(asnHeader)
|
||||
pr.Out.Header.Del(countryHeader)
|
||||
|
||||
if rq.h.config.AddLookupHeaders {
|
||||
setLookupHeaders(pr.Out.Header, rq.line.ASN, rq.line.Country)
|
||||
}
|
||||
}
|
||||
|
||||
// modifyResponse looks at the app's answer before ReverseProxy passes it
|
||||
@@ -474,7 +488,10 @@ func timing(start, end time.Time) *float64 {
|
||||
}
|
||||
|
||||
// addToHistory adds the request, which has ended, to its client's
|
||||
// history.
|
||||
// history, and then, for a client that was looked up, the answer kept
|
||||
// about it to that history and to the notes of the bans on its netblock:
|
||||
// an answer that came during the request may have come before either was
|
||||
// there, and one that comes later is added when it comes.
|
||||
func (rq *request) addToHistory() {
|
||||
var requestBytes int64
|
||||
if rq.body != nil {
|
||||
@@ -482,9 +499,9 @@ func (rq *request) addToHistory() {
|
||||
}
|
||||
|
||||
forwarded := !rq.upstreamStart.IsZero()
|
||||
group := clientGroup(rq.client)
|
||||
|
||||
rq.h.limiter.AddToHistory(clientGroup(rq.client), rq.h.now(), ratelimit.Request{
|
||||
Country: rq.line.Country,
|
||||
rq.h.limiter.AddToHistory(group, rq.h.now(), ratelimit.Request{
|
||||
Forwarded: forwarded,
|
||||
Refused: !forwarded && rq.refused.Load() != nil,
|
||||
Status: rq.out.status,
|
||||
@@ -492,6 +509,15 @@ func (rq *request) addToHistory() {
|
||||
ResponseBytes: rq.out.bytes,
|
||||
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
|
||||
})
|
||||
|
||||
if !rq.lookedUp {
|
||||
return
|
||||
}
|
||||
|
||||
answer, kept := rq.h.geojs.Kept(group)
|
||||
if kept {
|
||||
rq.h.addLookup(answer)
|
||||
}
|
||||
}
|
||||
|
||||
// clientRequestDeadline is when the client must have sent its whole
|
||||
|
||||
@@ -16,10 +16,10 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||
start := midnight()
|
||||
|
||||
for i, r := range []ratelimit.Request{
|
||||
{Country: "DE", Forwarded: true, Status: 200, RequestBytes: 10, ResponseBytes: 100},
|
||||
{Forwarded: true, Status: 200, RequestBytes: 10, ResponseBytes: 100},
|
||||
{Forwarded: true, Status: 101},
|
||||
{Forwarded: true, Status: 304, RequestBytes: 5},
|
||||
{Country: "FR", Refused: true, Status: 403, ResponseBytes: 10, BrokeLimit: true},
|
||||
{Refused: true, Status: 403, ResponseBytes: 10, BrokeLimit: true},
|
||||
{Forwarded: true, Status: 502, ResponseBytes: 12},
|
||||
// Closed without an answer: refused, and no response.
|
||||
{Refused: true, Status: 0},
|
||||
@@ -33,8 +33,6 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||
want := ratelimit.History{
|
||||
FirstSeen: start,
|
||||
LastSeen: start.Add(6 * time.Minute),
|
||||
Country: "FR",
|
||||
LookedUp: start.Add(3 * time.Minute),
|
||||
Requests: 7,
|
||||
Forwarded: 4,
|
||||
Refused: 2,
|
||||
@@ -52,6 +50,43 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
limiter := ratelimit.New(ratelimit.Limits{})
|
||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||
start := midnight()
|
||||
|
||||
limiter.AddToHistory(client, start, ratelimit.Request{Forwarded: true})
|
||||
limiter.AddLookup(client, start, "AS64496", "Example Net", "DE")
|
||||
|
||||
// A later answer replaces it, and one for a client the table does not
|
||||
// hold adds no client.
|
||||
limiter.AddLookup(client, start.Add(time.Hour), "AS64497", "Other Net", "FR")
|
||||
limiter.AddLookup(other, start, "AS64496", "Example Net", "DE")
|
||||
|
||||
want := ratelimit.History{
|
||||
FirstSeen: start,
|
||||
LastSeen: start,
|
||||
ASN: "AS64497",
|
||||
ASName: "Other Net",
|
||||
Country: "FR",
|
||||
LookedUp: start.Add(time.Hour),
|
||||
Requests: 1,
|
||||
Forwarded: 1,
|
||||
}
|
||||
|
||||
got := historyOf(t, limiter, client)
|
||||
if got != want {
|
||||
t.Errorf("history\n%+v\nwant\n%+v", got, want)
|
||||
}
|
||||
|
||||
if clients := limiter.Snapshot(); len(clients) != 1 {
|
||||
t.Errorf("the table holds %+v, want %s alone", clients, client)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetKeepsTheHistory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -66,8 +66,12 @@ type Buckets struct {
|
||||
type History struct {
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
// Country is the client's country as it was last looked up, and
|
||||
// LookedUp when that was; both are empty while it never was.
|
||||
// ASN, ASName and Country are the client's AS number, AS name and
|
||||
// country as last looked up, each empty when the lookup could not
|
||||
// find it, and LookedUp is when GeoJS gave that answer; all are empty
|
||||
// while the client never was looked up.
|
||||
ASN string `json:"asn,omitempty"`
|
||||
ASName string `json:"as_name,omitempty"`
|
||||
Country string `json:"country,omitempty"`
|
||||
LookedUp time.Time `json:"looked_up,omitzero"`
|
||||
// Requests are all the client's requests: Forwarded those passed to
|
||||
@@ -103,8 +107,6 @@ type Offences struct {
|
||||
|
||||
// Request is what a client's history keeps of one of its requests.
|
||||
type Request struct {
|
||||
// Country is the client's country, when the request looked it up.
|
||||
Country string
|
||||
// Forwarded is true for a request passed to the app, Refused for one
|
||||
// refused before anything reached it, a 401 at smallwebwaf's own
|
||||
// endpoints included. Both are false for any other request smallwebwaf
|
||||
@@ -209,11 +211,6 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) {
|
||||
|
||||
h.LastSeen = now
|
||||
|
||||
if r.Country != "" {
|
||||
h.Country = r.Country
|
||||
h.LookedUp = now
|
||||
}
|
||||
|
||||
h.Requests++
|
||||
if r.Forwarded {
|
||||
h.Forwarded++
|
||||
@@ -232,6 +229,24 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// AddLookup gives client's history its AS number, AS name and country, as
|
||||
// GeoJS gave them at lookedUp, if the table of clients holds the client.
|
||||
// It does not make the client the most recently seen.
|
||||
func (l *Limiter) AddLookup(
|
||||
client netip.Prefix, lookedUp time.Time, asn, asName, country string,
|
||||
) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
c, held := l.clients.Peek(client)
|
||||
if !held {
|
||||
return
|
||||
}
|
||||
|
||||
h := &c.History
|
||||
h.ASN, h.ASName, h.Country, h.LookedUp = asn, asName, country, lookedUp
|
||||
}
|
||||
|
||||
// Requests returns how many requests the clients inside netblock have
|
||||
// sent, as their histories count them.
|
||||
func (l *Limiter) Requests(netblock netip.Prefix) int64 {
|
||||
|
||||
@@ -80,11 +80,14 @@ type Line struct {
|
||||
// Request detail. RequestID is the X-Request-ID a trusted proxy sent,
|
||||
// or a new one, and is sent on to the app. ForwardedFor is the
|
||||
// X-Forwarded-For header as received. ClientGroup is the netblock the
|
||||
// client is counted as.
|
||||
// client is counted as. ASN, ASName and Country are the client's AS
|
||||
// number, AS name and country, as looked up.
|
||||
RequestID string `json:"request_id"`
|
||||
PeerIP string `json:"peer_ip"`
|
||||
ForwardedFor string `json:"forwarded_for,omitempty"`
|
||||
ClientGroup string `json:"client_group"`
|
||||
ASN string `json:"asn"`
|
||||
ASName string `json:"as_name"`
|
||||
Country string `json:"country"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
// ContentLength is the length of its body the request announced.
|
||||
|
||||
@@ -104,12 +104,16 @@ func (o *output) text() string {
|
||||
}
|
||||
|
||||
// run runs smallwebwaf with the settings in env until ctx is done, and
|
||||
// returns its exit status.
|
||||
// returns its exit status. SWWAF_LOOKUP_SOURCE is off unless env sets it,
|
||||
// so that no test sends GeoJS its clients' addresses.
|
||||
func run(ctx context.Context, env map[string]string, out *output) int {
|
||||
return smallwebwaf.Run(ctx, smallwebwaf.Params{
|
||||
Version: testVersion,
|
||||
LookupEnv: func(name string) (string, bool) {
|
||||
value, ok := env[name]
|
||||
if !ok && name == "SWWAF_LOOKUP_SOURCE" {
|
||||
return "off", true
|
||||
}
|
||||
|
||||
return value, ok
|
||||
},
|
||||
|
||||
@@ -35,6 +35,9 @@ const (
|
||||
clientsJSON = "clients.json"
|
||||
lookupsJSON = "lookups.json"
|
||||
alertsJSON = "alerts.json"
|
||||
// The AS number and AS name the tests' clients are looked up in.
|
||||
asn = "AS64496"
|
||||
asName = "Example Net"
|
||||
// What the process log says once Watch watches the directory, and as
|
||||
// it takes in an edit.
|
||||
watching = "watching the state files for edits"
|
||||
@@ -55,6 +58,8 @@ const permanentBansJSON = `{
|
||||
"cause": "admin",
|
||||
"reason": "scrapes every commit",
|
||||
"notes": {
|
||||
"asn": "AS64496",
|
||||
"as_name": "Example Net",
|
||||
"country": "DE",
|
||||
"limit": 1000,
|
||||
"window": "minute",
|
||||
@@ -1302,7 +1307,9 @@ func fill(params state.Params) {
|
||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||
|
||||
params.Ledger.Load([]bans.Ban{permanentBan()})
|
||||
params.Ledger.BanForLimit(client, now, bans.Notes{Country: "DE", Limit: 1})
|
||||
params.Ledger.BanForLimit(client, now, bans.Notes{
|
||||
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
|
||||
})
|
||||
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
|
||||
bans.Notes{RuleID: "env-file", Target: "path"})
|
||||
|
||||
@@ -1311,11 +1318,15 @@ func fill(params state.Params) {
|
||||
}
|
||||
|
||||
params.Limiter.AddToHistory(client, now, ratelimit.Request{
|
||||
Country: "DE", Forwarded: true, Status: 200, RequestBytes: 3, ResponseBytes: 5,
|
||||
Forwarded: true, Status: 200, RequestBytes: 3, ResponseBytes: 5,
|
||||
})
|
||||
params.Limiter.AddLookup(client, now.Add(-time.Hour), asn, asName, "DE")
|
||||
|
||||
params.GeoJS.Load([]lookup.Answer{
|
||||
{Client: client, Country: "DE", Answered: now.Add(-time.Hour), Used: now},
|
||||
{
|
||||
Client: client, ASN: asn, ASName: asName, Country: "DE",
|
||||
Answered: now.Add(-time.Hour), Used: now,
|
||||
},
|
||||
{
|
||||
Client: netip.MustParsePrefix("192.0.2.1/32"),
|
||||
Answered: now.Add(-time.Hour), Used: now.Add(-time.Minute),
|
||||
@@ -1350,6 +1361,8 @@ func permanentBan() bans.Ban {
|
||||
Cause: bans.CauseAdmin,
|
||||
Reason: "scrapes every commit",
|
||||
Notes: bans.Notes{
|
||||
ASN: asn,
|
||||
ASName: asName,
|
||||
Country: "DE",
|
||||
Limit: 1000,
|
||||
Window: "minute",
|
||||
|
||||
Reference in New Issue
Block a user