Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6dd46638a2 | ||
|
|
09260591c8 | ||
|
|
13e4cdcf24 | ||
|
|
419ee1c441 | ||
|
|
aaa28f541b | ||
|
|
98d7432077 | ||
|
|
11ac8aee50 | ||
|
|
e7cc0adbdb | ||
|
|
805c64a256 | ||
|
|
967ccdf75e |
No files matched your search
@@ -906,6 +906,8 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||
`internal/alerts`: one check or source each.
|
||||
- `internal/dirwatch`: watches a directory, so that `internal/rules` and
|
||||
`internal/lookup` read their files again when they change.
|
||||
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
||||
`internal/metrics`: the state and cache files, the log lines, the syslog
|
||||
sender and the metrics.
|
||||
|
||||
@@ -93,6 +93,9 @@ never as a time.
|
||||
imports no other package of this repository: `waf` gives it what it needs
|
||||
to count and report failures.
|
||||
- `internal/lookup`: looks clients up in the lookup database.
|
||||
- `internal/dirwatch`: watches a directory, and calls a function once it has
|
||||
gone a short time without a change. `internal/rules` and `internal/lookup`
|
||||
read their files again this way.
|
||||
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
||||
AbuseIPDB in the background, and keeps their answers. It also fetches
|
||||
AbuseIPDB's blacklist, which `internal/waf` bans.
|
||||
|
||||
@@ -13,6 +13,7 @@ require (
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/spf13/viper v1.21.0
|
||||
go.uber.org/fx v1.24.0
|
||||
golang.org/x/net v0.60.0
|
||||
golang.org/x/sys v0.48.0
|
||||
)
|
||||
|
||||
@@ -50,7 +51,6 @@ require (
|
||||
go.uber.org/zap v1.26.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
||||
golang.org/x/net v0.60.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
|
||||
@@ -19,6 +19,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// off is the value that switches a timeout, a size limit or a rate limit
|
||||
@@ -40,9 +42,7 @@ var (
|
||||
errNotPositive = errors.New("must be more than zero, or off")
|
||||
errNotMilliseconds = errors.New(
|
||||
"is not a whole number of milliseconds above zero, such as 2000")
|
||||
errEmptyItem = errors.New("has an empty item in its list")
|
||||
errNotNetblock = errors.New(
|
||||
"is not a netblock such as 10.0.0.0/8, or an address")
|
||||
errEmptyItem = errors.New("has an empty item in its list")
|
||||
errNotListenAddr = errors.New(
|
||||
"is not an address to listen on, such as :8080")
|
||||
errNotCommand = errors.New("is not rproxy or waf")
|
||||
@@ -253,7 +253,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
||||
netblocks := make([]netip.Prefix, 0, len(items))
|
||||
|
||||
for _, item := range items {
|
||||
netblock, err := ParseNetblock(item)
|
||||
netblock, err := reputation.ParseNetblock(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -264,27 +264,6 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
||||
return netblocks, nil
|
||||
}
|
||||
|
||||
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
||||
// address is a netblock of that address alone, a /32 or a /128. A
|
||||
// blocklist's lines are read with it too.
|
||||
func ParseNetblock(value string) (netip.Prefix, error) {
|
||||
if strings.Contains(value, "/") {
|
||||
netblock, err := netip.ParsePrefix(value)
|
||||
if err != nil {
|
||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||
}
|
||||
|
||||
return netblock.Masked(), nil
|
||||
}
|
||||
|
||||
addr, err := netip.ParseAddr(value)
|
||||
if err != nil || addr.Zone() != "" {
|
||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||
}
|
||||
|
||||
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
||||
}
|
||||
|
||||
// parseListenAddr checks an address to listen on: an optional host and a
|
||||
// port number.
|
||||
func parseListenAddr(value string) (string, error) {
|
||||
|
||||
+169
-152
@@ -20,11 +20,26 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"golang.org/x/net/http/httpguts"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// The names of the settings that waf writes into log lines and ban notes,
|
||||
// as the setting that refused a request or lowered a client's limits.
|
||||
const (
|
||||
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
|
||||
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
|
||||
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
|
||||
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
|
||||
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
|
||||
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
||||
)
|
||||
|
||||
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
||||
@@ -75,7 +90,7 @@ type WAF struct {
|
||||
BytesLimitPerMinute int64
|
||||
BytesLimitPerHour int64
|
||||
BytesLimitPerDay int64
|
||||
BytesCounted string
|
||||
BytesCounted BytesCounted
|
||||
// LookupSource is where each client's AS number and country are looked
|
||||
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
|
||||
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
|
||||
@@ -86,7 +101,7 @@ type WAF struct {
|
||||
// AddLookupHeaders is true when the app is passed the client's AS number
|
||||
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
|
||||
// LookupSource is off.
|
||||
LookupSource string
|
||||
LookupSource LookupSource
|
||||
LookupDBPath string
|
||||
ASNAndCountryLookupTimeout time.Duration
|
||||
AddLookupHeaders bool
|
||||
@@ -100,7 +115,7 @@ type WAF struct {
|
||||
// with a tag that percentage of every rate limit and byte limit, from 1
|
||||
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
||||
// lower case, a country code in capitals.
|
||||
TagLists []TagList
|
||||
TagLists []reputation.TagList
|
||||
TagDeny []string
|
||||
TagLimitPercent map[string]int64
|
||||
// RouteWatchURL is where the routewatch instance is that adds the asn=
|
||||
@@ -131,7 +146,7 @@ type WAF struct {
|
||||
// BlocklistLimitPercent is the percentage of every limit it gets.
|
||||
BlocklistURLs []string
|
||||
BlocklistRefresh time.Duration
|
||||
BlocklistAction string
|
||||
BlocklistAction ReputationAction
|
||||
BlocklistLimitPercent int64
|
||||
// DNSBLZones are the DNSBL zones clients are asked about
|
||||
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
|
||||
@@ -154,7 +169,7 @@ type WAF struct {
|
||||
AbuseIPDBKey string
|
||||
AbuseIPDBMinScore int64
|
||||
AbuseIPDBDailyBudget int
|
||||
DNSBLAbuseIPDBAction string
|
||||
DNSBLAbuseIPDBAction ReputationAction
|
||||
DNSBLAbuseIPDBLimitPercent int64
|
||||
IPReputationCacheTTL time.Duration
|
||||
DNSBLAbuseIPDBTimeout time.Duration
|
||||
@@ -237,7 +252,7 @@ type WAF struct {
|
||||
// reads none.
|
||||
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
|
||||
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
|
||||
CoreRuleSetMode string
|
||||
CoreRuleSetMode CoreRuleSetMode
|
||||
CoreRuleSetParanoiaLevel int
|
||||
CoreRuleSetScoreThreshold int
|
||||
CoreRuleSetDisabledRules []int
|
||||
@@ -316,20 +331,66 @@ type WAF struct {
|
||||
settings []slog.Attr
|
||||
}
|
||||
|
||||
// BytesCounted is which body bytes count toward the byte limits and the
|
||||
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
|
||||
type BytesCounted string
|
||||
|
||||
// The values of SWW_WAF_BYTES_COUNTED.
|
||||
const (
|
||||
// BytesCountedResponse counts the response's body bytes.
|
||||
BytesCountedResponse BytesCounted = "response"
|
||||
// BytesCountedRequest counts the request's body bytes.
|
||||
BytesCountedRequest BytesCounted = "request"
|
||||
// BytesCountedBoth counts the body bytes of both.
|
||||
BytesCountedBoth BytesCounted = "both"
|
||||
)
|
||||
|
||||
// LookupSource is where each client's AS number and country are looked up,
|
||||
// a value of SWW_WAF_LOOKUP_SOURCE.
|
||||
type LookupSource string
|
||||
|
||||
// The values of SWW_WAF_LOOKUP_SOURCE.
|
||||
const (
|
||||
// LookupSourceGeoJS asks GeoJS.
|
||||
LookupSourceGeoJS LookupSource = "geojs"
|
||||
// LookupSourceFile looks clients up in the lookup database, the file
|
||||
// SWW_WAF_LOOKUP_DB_PATH names.
|
||||
LookupSourceFile LookupSource = "file"
|
||||
// LookupSourceOff looks no client up.
|
||||
LookupSourceOff LookupSource = "off"
|
||||
)
|
||||
|
||||
// ReputationAction is what is done with a client a reputation source
|
||||
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
|
||||
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
||||
type ReputationAction string
|
||||
|
||||
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
||||
const (
|
||||
// ReputationActionDeny refuses the client's requests.
|
||||
ReputationActionDeny ReputationAction = "deny"
|
||||
// ReputationActionLimit gives the client a percentage of every limit,
|
||||
// written limit:<percent> in the setting.
|
||||
ReputationActionLimit ReputationAction = "limit"
|
||||
// ReputationActionLog only notes the hit in the log line and raises an
|
||||
// alert.
|
||||
ReputationActionLog ReputationAction = "log"
|
||||
)
|
||||
|
||||
// CoreRuleSetMode is what the Core Rule Set does, a value of
|
||||
// SWW_WAF_CORE_RULE_SET_MODE.
|
||||
type CoreRuleSetMode string
|
||||
|
||||
// The values of SWW_WAF_CORE_RULE_SET_MODE.
|
||||
const (
|
||||
// CoreRuleSetModeOff runs no request through the Core Rule Set.
|
||||
CoreRuleSetModeOff = off
|
||||
CoreRuleSetModeOff CoreRuleSetMode = "off"
|
||||
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
|
||||
CoreRuleSetModeDetect = "detect"
|
||||
CoreRuleSetModeDetect CoreRuleSetMode = "detect"
|
||||
// CoreRuleSetModeBlock refuses a match with 403.
|
||||
CoreRuleSetModeBlock = "block"
|
||||
CoreRuleSetModeBlock CoreRuleSetMode = "block"
|
||||
)
|
||||
|
||||
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
|
||||
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
|
||||
const fileSource = "file"
|
||||
|
||||
const (
|
||||
day = 24 * time.Hour
|
||||
ipv4Bits = 32
|
||||
@@ -451,12 +512,8 @@ var (
|
||||
"is not a whole number above zero, such as 60, or off",
|
||||
)
|
||||
errNotUTF8 = errors.New("is not valid UTF-8")
|
||||
errNotASN = errors.New("is not an AS number such as AS64496")
|
||||
errNotPercentItem = errors.New(
|
||||
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
|
||||
errNotPercent = errors.New(
|
||||
"is not a percentage, a whole number from 0 to 100",
|
||||
)
|
||||
errNotAbuseIPDBThreshold = errors.New(
|
||||
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
||||
"blacklist can be cut at",
|
||||
@@ -558,10 +615,13 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
||||
),
|
||||
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
||||
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
|
||||
BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
|
||||
BytesCounted: env.bytesCounted(
|
||||
"SWW_WAF_BYTES_COUNTED",
|
||||
BytesCountedBoth,
|
||||
),
|
||||
LookupSource: env.lookupSource(
|
||||
"SWW_WAF_LOOKUP_SOURCE",
|
||||
"geojs",
|
||||
LookupSourceGeoJS,
|
||||
),
|
||||
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
|
||||
ASNAndCountryLookupTimeout: env.milliseconds(
|
||||
@@ -569,24 +629,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
||||
"800",
|
||||
),
|
||||
ExclusivelyAllowedCountries: env.countries(
|
||||
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
||||
ExclusivelyAllowedCountriesSetting, ""),
|
||||
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
||||
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
||||
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
|
||||
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
|
||||
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
||||
ASNBytesLimitPercent: env.percents(
|
||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
||||
ParseASN,
|
||||
ASNBytesLimitPercentSetting,
|
||||
reputation.ParseASN,
|
||||
),
|
||||
CountryBytesLimitPercent: env.percents(
|
||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
||||
CountryBytesLimitPercentSetting,
|
||||
parseCountry,
|
||||
),
|
||||
UnknownCountryLimitPercent: env.percent(
|
||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
||||
UnknownCountryLimitPercentSetting,
|
||||
"100",
|
||||
),
|
||||
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
|
||||
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
|
||||
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
||||
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
||||
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
||||
@@ -758,11 +818,11 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
||||
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
||||
// The app gets the headers by default, unless no client is looked up.
|
||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||
strconv.FormatBool(cfg.LookupSource != off))
|
||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||
"SWW_WAF_BLOCKLIST_ACTION", "deny")
|
||||
BlocklistActionSetting, string(ReputationActionDeny))
|
||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
||||
DNSBLAbuseIPDBActionSetting, "limit:25")
|
||||
|
||||
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
||||
env.checkLookupDBPath(cfg)
|
||||
@@ -858,9 +918,12 @@ func (e *environment) count(name, defaultValue string) int64 {
|
||||
|
||||
// bytesCounted reads the setting that is which body bytes count toward the
|
||||
// byte limits and the byte anomaly thresholds: response, request or both.
|
||||
func (e *environment) bytesCounted(name, defaultValue string) string {
|
||||
value := e.value(name, defaultValue)
|
||||
if value != "response" && value != "request" && value != "both" {
|
||||
func (e *environment) bytesCounted(
|
||||
name string, defaultValue BytesCounted,
|
||||
) BytesCounted {
|
||||
value := BytesCounted(e.value(name, string(defaultValue)))
|
||||
if value != BytesCountedResponse && value != BytesCountedRequest &&
|
||||
value != BytesCountedBoth {
|
||||
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
|
||||
}
|
||||
|
||||
@@ -905,8 +968,10 @@ func (e *environment) trapPaths(name string) []string {
|
||||
|
||||
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
|
||||
// detect or block.
|
||||
func (e *environment) coreRuleSetMode(name, defaultValue string) string {
|
||||
mode := e.value(name, defaultValue)
|
||||
func (e *environment) coreRuleSetMode(
|
||||
name string, defaultValue CoreRuleSetMode,
|
||||
) CoreRuleSetMode {
|
||||
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
|
||||
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
|
||||
mode != CoreRuleSetModeBlock {
|
||||
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
|
||||
@@ -957,7 +1022,7 @@ func (e *environment) percents(
|
||||
}
|
||||
|
||||
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
|
||||
func (e *environment) tagLists(name string) []TagList {
|
||||
func (e *environment) tagLists(name string) []reputation.TagList {
|
||||
lists, err := parseTagLists(e.value(name, ""))
|
||||
e.check(name, err)
|
||||
|
||||
@@ -983,7 +1048,7 @@ func (e *environment) tagPercents(name string) map[string]int64 {
|
||||
|
||||
// percent reads a setting that is a percentage, from 0 to 100.
|
||||
func (e *environment) percent(name, defaultValue string) int64 {
|
||||
percent, err := ParsePercent(e.value(name, defaultValue))
|
||||
percent, err := reputation.ParsePercent(e.value(name, defaultValue))
|
||||
e.check(name, err)
|
||||
|
||||
return percent
|
||||
@@ -1056,25 +1121,29 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
|
||||
// action reads a setting that is what is done with a client a blocklist
|
||||
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
||||
// as limit and the percentage.
|
||||
func (e *environment) action(name, defaultValue string) (string, int64) {
|
||||
value := e.value(name, defaultValue)
|
||||
if value == "deny" || value == "log" {
|
||||
func (e *environment) action(
|
||||
name, defaultValue string,
|
||||
) (ReputationAction, int64) {
|
||||
value := ReputationAction(e.value(name, defaultValue))
|
||||
if value == ReputationActionDeny || value == ReputationActionLog {
|
||||
return value, 0
|
||||
}
|
||||
|
||||
percentText, isLimit := strings.CutPrefix(value, "limit:")
|
||||
percentText, isLimit := strings.CutPrefix(
|
||||
string(value), string(ReputationActionLimit)+":",
|
||||
)
|
||||
|
||||
percent, err := ParsePercent(percentText)
|
||||
percent, err := reputation.ParsePercent(percentText)
|
||||
if !isLimit || err != nil {
|
||||
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
||||
}
|
||||
|
||||
return "limit", percent
|
||||
return ReputationActionLimit, percent
|
||||
}
|
||||
|
||||
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
||||
// default. The log shows each zone with its key masked, as MaskZoneKey
|
||||
// masks it.
|
||||
// default. The log shows each zone with its key masked, as
|
||||
// reputation.MaskZoneKey masks it.
|
||||
func (e *environment) zones(name string) []string {
|
||||
value, _ := e.lookup(name)
|
||||
zones, err := parseZones(value)
|
||||
@@ -1082,7 +1151,7 @@ func (e *environment) zones(name string) []string {
|
||||
|
||||
logged := make([]string, len(zones))
|
||||
for i, zone := range zones {
|
||||
logged[i] = MaskZoneKey(zone)
|
||||
logged[i] = reputation.MaskZoneKey(zone)
|
||||
}
|
||||
|
||||
e.settings = append(
|
||||
@@ -1120,9 +1189,12 @@ func (e *environment) routeWatchURL(name string) string {
|
||||
|
||||
// lookupSource reads the setting that is where clients are looked up:
|
||||
// geojs, file, or off.
|
||||
func (e *environment) lookupSource(name, defaultValue string) string {
|
||||
source := e.value(name, defaultValue)
|
||||
if source != "geojs" && source != fileSource && source != off {
|
||||
func (e *environment) lookupSource(
|
||||
name string, defaultValue LookupSource,
|
||||
) LookupSource {
|
||||
source := LookupSource(e.value(name, string(defaultValue)))
|
||||
if source != LookupSourceGeoJS && source != LookupSourceFile &&
|
||||
source != LookupSourceOff {
|
||||
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
||||
}
|
||||
|
||||
@@ -1134,12 +1206,12 @@ func (e *environment) lookupSource(name, defaultValue string) string {
|
||||
// one source at a time.
|
||||
func (e *environment) checkLookupDBPath(cfg *WAF) {
|
||||
switch {
|
||||
case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
|
||||
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "":
|
||||
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
|
||||
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
|
||||
errNeedsDBPath,
|
||||
))
|
||||
case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
|
||||
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "":
|
||||
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
|
||||
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
|
||||
cfg.LookupSource,
|
||||
@@ -1159,12 +1231,12 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
|
||||
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
||||
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||
e.check(ExclusivelyAllowedCountriesSetting,
|
||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.LookupSource != off {
|
||||
if cfg.LookupSource != LookupSourceOff {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1178,15 +1250,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||
name string
|
||||
set bool
|
||||
}{
|
||||
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
||||
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
||||
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
|
||||
{TagLimitPercentSetting, slices.ContainsFunc(
|
||||
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
||||
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
|
||||
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
|
||||
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
|
||||
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
|
||||
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
|
||||
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
|
||||
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
|
||||
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
|
||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
||||
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
||||
@@ -1206,7 +1278,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||
// blocklist too: the file at a URL is fetched as one list or the other.
|
||||
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
||||
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
||||
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
|
||||
e.check(ASNLimitPercentURLSetting,
|
||||
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
||||
}
|
||||
}
|
||||
@@ -1233,14 +1305,18 @@ func (e *environment) checkTags(cfg *WAF) {
|
||||
}
|
||||
|
||||
for tag := range cfg.TagLimitPercent {
|
||||
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
|
||||
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
|
||||
}
|
||||
}
|
||||
|
||||
// checkTag returns an error when tag would never apply, as checkTags says.
|
||||
func checkTag(cfg *WAF, tag string) error {
|
||||
if !requestCanGetTag(cfg, tag) {
|
||||
return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
|
||||
return fmt.Errorf(
|
||||
"%q %w",
|
||||
reputation.MaskZoneKey(tag),
|
||||
errNoRequestGetsTag,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -1248,7 +1324,7 @@ func checkTag(cfg *WAF, tag string) error {
|
||||
|
||||
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
||||
func requestCanGetTag(cfg *WAF, tag string) bool {
|
||||
isListName := func(list TagList) bool { return list.Name == tag }
|
||||
isListName := func(list reputation.TagList) bool { return list.Name == tag }
|
||||
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
|
||||
return true
|
||||
}
|
||||
@@ -1272,7 +1348,7 @@ func requestCanGetTag(cfg *WAF, tag string) bool {
|
||||
|
||||
return isZoneTag &&
|
||||
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
||||
return strings.ToLower(MaskZoneKey(each)) == zone
|
||||
return strings.ToLower(reputation.MaskZoneKey(each)) == zone
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1770,7 +1846,9 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
|
||||
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
|
||||
}
|
||||
|
||||
netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
|
||||
netblock, err := reputation.ParseNetblock(
|
||||
strings.TrimSpace(netblockText),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1976,21 +2054,6 @@ func parseCountry(value string) (string, error) {
|
||||
return country, nil
|
||||
}
|
||||
|
||||
// ParseASN reads an AS number such as AS64496, in either case, and
|
||||
// returns it as the lookup gives it: AS and the number, in capitals and
|
||||
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
||||
// read with it too.
|
||||
func ParseASN(value string) (string, error) {
|
||||
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
||||
|
||||
number, err := strconv.ParseUint(digits, 10, 32)
|
||||
if !hasAS || err != nil {
|
||||
return "", fmt.Errorf("%q %w", value, errNotASN)
|
||||
}
|
||||
|
||||
return "AS" + strconv.FormatUint(number, 10), nil
|
||||
}
|
||||
|
||||
// The tags a request can get that the settings can name, as waf
|
||||
// writes them. The prefixes are followed by a value.
|
||||
const (
|
||||
@@ -2004,24 +2067,17 @@ const (
|
||||
ExploitSignatureTag = "exploitsignature"
|
||||
)
|
||||
|
||||
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
||||
// requests of the clients it holds, and the URL it is fetched from.
|
||||
type TagList struct {
|
||||
Name string
|
||||
URL string
|
||||
}
|
||||
|
||||
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
||||
// torexit=https://example.com/tor.txt, and returns each list with its name
|
||||
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
||||
// = before http. A name or a URL listed twice is an error.
|
||||
func parseTagLists(value string) ([]TagList, error) {
|
||||
func parseTagLists(value string) ([]reputation.TagList, error) {
|
||||
items, err := parseList(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
lists := make([]TagList, 0, len(items))
|
||||
lists := make([]reputation.TagList, 0, len(items))
|
||||
|
||||
for _, item := range items {
|
||||
end := strings.Index(item, "=http")
|
||||
@@ -2029,9 +2085,12 @@ func parseTagLists(value string) ([]TagList, error) {
|
||||
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
||||
}
|
||||
|
||||
list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]}
|
||||
list := reputation.TagList{
|
||||
Name: strings.ToLower(item[:end]),
|
||||
URL: item[end+1:],
|
||||
}
|
||||
|
||||
if slices.ContainsFunc(lists, func(earlier TagList) bool {
|
||||
if slices.ContainsFunc(lists, func(earlier reputation.TagList) bool {
|
||||
return earlier.Name == list.Name || earlier.URL == list.URL
|
||||
}) {
|
||||
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
||||
@@ -2044,7 +2103,8 @@ func parseTagLists(value string) ([]TagList, error) {
|
||||
}
|
||||
|
||||
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
||||
// tag listed twice is an error, which shows the tag as MaskZoneKey does.
|
||||
// tag listed twice is an error, which shows the tag as
|
||||
// reputation.MaskZoneKey does.
|
||||
func parseTags(value string) ([]string, error) {
|
||||
items, err := parseList(value)
|
||||
if err != nil {
|
||||
@@ -2064,7 +2124,7 @@ func parseTags(value string) ([]string, error) {
|
||||
if slices.Contains(tags, tag) {
|
||||
return nil, fmt.Errorf(
|
||||
"%q %w",
|
||||
MaskZoneKey(item),
|
||||
reputation.MaskZoneKey(item),
|
||||
errListedTwice,
|
||||
)
|
||||
}
|
||||
@@ -2133,7 +2193,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
||||
if colon <= 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"%q %w",
|
||||
MaskZoneKey(item),
|
||||
reputation.MaskZoneKey(item),
|
||||
errNotTagPercentItem,
|
||||
)
|
||||
}
|
||||
@@ -2147,7 +2207,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
||||
if err != nil || percent < 1 || percent > 99 {
|
||||
return nil, fmt.Errorf(
|
||||
"%q %w",
|
||||
MaskZoneKey(item[colon+1:]),
|
||||
reputation.MaskZoneKey(item[colon+1:]),
|
||||
errNotTagPercent,
|
||||
)
|
||||
}
|
||||
@@ -2159,7 +2219,11 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
||||
if _, listed := percents[tag]; listed {
|
||||
// tag, since item[:colon] may end in a space, which MaskZoneKey
|
||||
// would not mask.
|
||||
return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
|
||||
return nil, fmt.Errorf(
|
||||
"%q %w",
|
||||
reputation.MaskZoneKey(tag),
|
||||
errListedTwice,
|
||||
)
|
||||
}
|
||||
|
||||
percents[tag] = percent
|
||||
@@ -2193,7 +2257,7 @@ func parsePercents(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
percent, err := ParsePercent(percentText)
|
||||
percent, err := reputation.ParsePercent(percentText)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -2208,38 +2272,6 @@ func parsePercents(
|
||||
return percents, nil
|
||||
}
|
||||
|
||||
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
||||
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
||||
func ParsePercent(value string) (int64, error) {
|
||||
percent, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || percent < 0 || percent > 100 {
|
||||
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
||||
}
|
||||
|
||||
return percent, nil
|
||||
}
|
||||
|
||||
// headerNameChars are the characters RFC 9110 allows in a header name:
|
||||
// letters, digits and these marks.
|
||||
const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
|
||||
"0123456789!#$%&'*+-.^_`|~"
|
||||
|
||||
// IsHeaderName reports whether name can be a header name: one or more of
|
||||
// the characters RFC 9110 allows in one.
|
||||
func IsHeaderName(name string) bool {
|
||||
if name == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, char := range name {
|
||||
if !strings.ContainsRune(headerNameChars, char) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// parseHeaderNames reads a comma-separated list of header names in either
|
||||
// case, and returns them in lower case. Host and Transfer-Encoding are
|
||||
// refused: Go's HTTP server takes them out of the request's headers.
|
||||
@@ -2252,7 +2284,7 @@ func parseHeaderNames(value string) ([]string, error) {
|
||||
headers := make([]string, 0, len(items))
|
||||
|
||||
for _, item := range items {
|
||||
if !IsHeaderName(item) {
|
||||
if !httpguts.ValidHeaderFieldName(item) {
|
||||
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
|
||||
}
|
||||
|
||||
@@ -2408,8 +2440,8 @@ const (
|
||||
// other name for one that does not exist, so that the zone would list no
|
||||
// client. A zone listed twice is an error, whatever the case of its
|
||||
// letters, which DNS names ignore, and whatever its key, since
|
||||
// MaskZoneKey shows two keys of one zone alike. An error shows a zone as
|
||||
// MaskZoneKey does.
|
||||
// reputation.MaskZoneKey shows two keys of one zone alike. An error shows a
|
||||
// zone as it does.
|
||||
func parseZones(value string) ([]string, error) {
|
||||
zones, err := parseList(value)
|
||||
if err != nil {
|
||||
@@ -2419,11 +2451,14 @@ func parseZones(value string) ([]string, error) {
|
||||
}
|
||||
|
||||
for i, zone := range zones {
|
||||
shown := MaskZoneKey(zone)
|
||||
shown := reputation.MaskZoneKey(zone)
|
||||
listedBefore := slices.ContainsFunc(
|
||||
zones[:i],
|
||||
func(earlier string) bool {
|
||||
return strings.EqualFold(MaskZoneKey(earlier), shown)
|
||||
return strings.EqualFold(
|
||||
reputation.MaskZoneKey(earlier),
|
||||
shown,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2440,24 +2475,6 @@ func parseZones(value string) ([]string, error) {
|
||||
return zones, nil
|
||||
}
|
||||
|
||||
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
||||
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
||||
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
||||
// other zone it returns as it is. A zone is shown so wherever it leaves
|
||||
// the process: in the log, the alerts and the metrics.
|
||||
func MaskZoneKey(zone string) string {
|
||||
// DNS names ignore case, and a name may be written with a dot at its
|
||||
// end.
|
||||
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
||||
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
||||
return zone
|
||||
}
|
||||
|
||||
_, rest, _ := strings.Cut(zone, ".")
|
||||
|
||||
return masked + "." + rest
|
||||
}
|
||||
|
||||
// isZone reports whether each label of zone is as parseZones takes it. It
|
||||
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
|
||||
func isZone(zone string) bool {
|
||||
@@ -2513,7 +2530,7 @@ func parseWebhookHeaders(value string) (http.Header, string, error) {
|
||||
name, headerValue, found := strings.Cut(item, ":")
|
||||
name = strings.TrimSpace(name)
|
||||
|
||||
if !found || !IsHeaderName(name) ||
|
||||
if !found || !httpguts.ValidHeaderFieldName(name) ||
|
||||
strings.ContainsAny(headerValue, "\r\n\x00") {
|
||||
return nil, "", fmt.Errorf(
|
||||
"item %d %w",
|
||||
|
||||
+35
-25
@@ -18,10 +18,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"golang.org/x/net/http/httpguts"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// The settings, by name.
|
||||
@@ -614,7 +616,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
|
||||
},
|
||||
{
|
||||
environment{
|
||||
coreRuleSetMode: config.CoreRuleSetModeDetect,
|
||||
coreRuleSetMode: string(config.CoreRuleSetModeDetect),
|
||||
coreRuleSetParanoiaLevel: "4",
|
||||
coreRuleSetScoreThreshold: "10",
|
||||
coreRuleSetDisabledRules: "942100, 920350",
|
||||
@@ -864,6 +866,32 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// config and rules check header names with httpguts.ValidHeaderFieldName.
|
||||
// It takes a name of one or more of the characters RFC 9110 allows in one,
|
||||
// as the check it replaced did: letters, digits and these marks.
|
||||
func TestHeaderNameCheckTakesTheRFC9110Characters(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const marks = "!#$%&'*+-.^_`|~"
|
||||
|
||||
if httpguts.ValidHeaderFieldName("") {
|
||||
t.Error("the empty name is taken")
|
||||
}
|
||||
|
||||
for b := range 256 {
|
||||
char := byte(b)
|
||||
want := 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' ||
|
||||
'0' <= char && char <= '9' || strings.IndexByte(marks, char) >= 0
|
||||
|
||||
// string(char) would be the UTF-8 of U+0080 to U+00FF, two bytes,
|
||||
// for a byte above 0x7f.
|
||||
name := string([]byte{char})
|
||||
if got := httpguts.ValidHeaderFieldName(name); got != want {
|
||||
t.Errorf("the name %q is taken: %t, want %t", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteLogSettingsDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1694,7 +1722,7 @@ func TestTagSettingsAsSet(t *testing.T) {
|
||||
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
||||
})
|
||||
|
||||
wantLists := []config.TagList{
|
||||
wantLists := []reputation.TagList{
|
||||
{Name: torExitTag, URL: torURL},
|
||||
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
||||
}
|
||||
@@ -2033,8 +2061,9 @@ func TestReputationSettingsAsSet(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
value, action string
|
||||
percent int64
|
||||
value string
|
||||
action config.ReputationAction
|
||||
percent int64
|
||||
}{
|
||||
{actionDeny, actionDeny, 0},
|
||||
{actionLog, actionLog, 0},
|
||||
@@ -2167,7 +2196,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
|
||||
// want are the zones, resolver, action and percent config.WAF gives.
|
||||
want []string
|
||||
wantResolver string
|
||||
wantAction string
|
||||
wantAction config.ReputationAction
|
||||
wantPercent int64
|
||||
}{
|
||||
{
|
||||
@@ -2287,25 +2316,6 @@ func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
for zone, want := range map[string]string{
|
||||
spamhaus: spamhausMasked,
|
||||
spamhaus + ".": spamhausMasked + ".",
|
||||
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
||||
dronebl: dronebl,
|
||||
"dq.spamhaus.net": "dq.spamhaus.net",
|
||||
spamhaus + ".example": spamhaus + ".example",
|
||||
} {
|
||||
if got := config.MaskZoneKey(zone); got != want {
|
||||
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -3049,7 +3059,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||
logLevel: "info",
|
||||
ruleFilesDir: "/etc/sww/rules.d",
|
||||
ruleFilesEnabled: "true",
|
||||
coreRuleSetMode: config.CoreRuleSetModeBlock,
|
||||
coreRuleSetMode: string(config.CoreRuleSetModeBlock),
|
||||
coreRuleSetParanoiaLevel: "1",
|
||||
coreRuleSetScoreThreshold: "5",
|
||||
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
// Package dirwatch watches a directory, and calls a function once the
|
||||
// directory has gone a short time without a change after one. The rule
|
||||
// files and the lookup database are read again this way.
|
||||
package dirwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
)
|
||||
|
||||
// quietTime is how long the directory must go without a change before the
|
||||
// function is called, so that a file still being written, such as one
|
||||
// saved in place, appended to or copied in with scp, is read only once
|
||||
// whole.
|
||||
const quietTime = 2 * time.Second
|
||||
|
||||
// Watcher watches one directory. The directory is watched, rather than a
|
||||
// file in it, so that a file replaced by a rename is seen.
|
||||
type Watcher struct {
|
||||
watcher *fsnotify.Watcher
|
||||
}
|
||||
|
||||
// New starts watching dir, or returns why it cannot.
|
||||
func New(dir string) (*Watcher, error) {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = watcher.Add(dir)
|
||||
if err != nil {
|
||||
_ = watcher.Close()
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Watcher{watcher: watcher}, nil
|
||||
}
|
||||
|
||||
// Close stops watching the directory.
|
||||
func (w *Watcher) Close() error {
|
||||
return w.watcher.Close()
|
||||
}
|
||||
|
||||
// CallWhenQuiet calls changed each time the directory has gone quietTime
|
||||
// without a change, after a change to a path for which counts is true, until
|
||||
// ctx is done. It also calls failed with each error the watch reports. The
|
||||
// wait starts at once, as if for a change, so that a change made before the
|
||||
// watch began is taken in too.
|
||||
func (w *Watcher) CallWhenQuiet(
|
||||
ctx context.Context,
|
||||
counts func(path string) bool,
|
||||
changed func(),
|
||||
failed func(error),
|
||||
) {
|
||||
callWhenQuiet(
|
||||
ctx, w.watcher.Events, w.watcher.Errors, counts, changed, failed,
|
||||
)
|
||||
}
|
||||
|
||||
// callWhenQuiet is CallWhenQuiet, for the changes from events and the
|
||||
// errors from errs.
|
||||
func callWhenQuiet(
|
||||
ctx context.Context,
|
||||
events <-chan fsnotify.Event,
|
||||
errs <-chan error,
|
||||
counts func(path string) bool,
|
||||
changed func(),
|
||||
failed func(error),
|
||||
) {
|
||||
quiet := time.NewTimer(quietTime)
|
||||
defer quiet.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case event := <-events:
|
||||
if counts(event.Name) {
|
||||
quiet.Reset(quietTime)
|
||||
}
|
||||
case <-quiet.C:
|
||||
changed()
|
||||
case err := <-errs:
|
||||
failed(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package dirwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
)
|
||||
|
||||
// errWatch is an error the watch reports.
|
||||
var errWatch = errors.New("too many changes")
|
||||
|
||||
func TestDirectoryThatCannotBeWatchedIsAnError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := New(filepath.Join(t.TempDir(), "missing"))
|
||||
if err == nil {
|
||||
t.Error("a missing directory is watched, want an error")
|
||||
}
|
||||
}
|
||||
|
||||
// The tests below run callWhenQuiet in a synctest bubble, where time is a
|
||||
// clock of the test's own: time.Sleep moves it on at once, and
|
||||
// synctest.Wait returns once callWhenQuiet waits again, so that every call
|
||||
// due by then is made. The test sends the changes itself, as the watch of
|
||||
// a directory cannot run in a bubble.
|
||||
|
||||
func TestSecondChangeStartsTheWaitAgain(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
loop := start(t, everyPath)
|
||||
|
||||
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 0)
|
||||
|
||||
// The second part starts the wait again.
|
||||
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 0)
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCalledOnceQuietAfterTheStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
// An edit saved before the watch began sends no change, and is
|
||||
// taken in by this call.
|
||||
loop := start(t, everyPath)
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 1)
|
||||
|
||||
// Long after, it has been called only once.
|
||||
time.Sleep(time.Hour)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestChangeThatDoesNotCountIsNoChange(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
loop := start(t, func(path string) bool {
|
||||
return path == "ipinfo_lite.mmdb"
|
||||
})
|
||||
|
||||
// The wait that starts with the watch ends with a call.
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
|
||||
loop.changes <- fsnotify.Event{Name: "other.mmdb", Op: fsnotify.Create}
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 1)
|
||||
|
||||
loop.changes <- fsnotify.Event{Name: "ipinfo_lite.mmdb", Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
loop.wantCalls(t, 2)
|
||||
})
|
||||
}
|
||||
|
||||
func TestErrorsOfTheWatchHandedOn(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
loop := start(t, everyPath)
|
||||
|
||||
loop.errs <- errWatch
|
||||
|
||||
synctest.Wait()
|
||||
|
||||
if !errors.Is(loop.failure, errWatch) {
|
||||
t.Errorf("failed with %v, want %v", loop.failure, errWatch)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// loop is a callWhenQuiet that a test runs.
|
||||
type loop struct {
|
||||
// changes and errs send it changes and errors.
|
||||
changes chan fsnotify.Event
|
||||
errs chan error
|
||||
// calls counts its calls of changed, and failure is the last error it
|
||||
// handed to failed.
|
||||
calls atomic.Int32
|
||||
failure error
|
||||
}
|
||||
|
||||
// start runs callWhenQuiet, with counts, until the test ends.
|
||||
func start(t *testing.T, counts func(path string) bool) *loop {
|
||||
t.Helper()
|
||||
|
||||
l := &loop{changes: make(chan fsnotify.Event), errs: make(chan error)}
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
callWhenQuiet(ctx, l.changes, l.errs, counts,
|
||||
func() { l.calls.Add(1) },
|
||||
func(err error) { l.failure = err },
|
||||
)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
<-stopped
|
||||
})
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
// everyPath counts a change to any path.
|
||||
func everyPath(string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// wantCalls checks how many times changed has been called.
|
||||
func (l *loop) wantCalls(t *testing.T, want int32) {
|
||||
t.Helper()
|
||||
|
||||
got := l.calls.Load()
|
||||
if got != want {
|
||||
t.Errorf("changed called %d times, want %d", got, want)
|
||||
}
|
||||
}
|
||||
+47
-14
@@ -99,8 +99,13 @@ type GeoJS struct {
|
||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||
// and those it is being asked about.
|
||||
waiting map[netip.Prefix]*wait
|
||||
// asking is true while a request to GeoJS is under way.
|
||||
asking bool
|
||||
// asking is true while a request to GeoJS is under way, and endAsking
|
||||
// ends it. stopped is true once StopAsking has been called, and
|
||||
// askGoroutines waits for the goroutine that makes the requests.
|
||||
asking bool
|
||||
endAsking context.CancelFunc
|
||||
stopped bool
|
||||
askGoroutines sync.WaitGroup
|
||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||
// zero after an answer; retryAt is when it may be asked again.
|
||||
retryDelay time.Duration
|
||||
@@ -210,6 +215,23 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
||||
return answer
|
||||
}
|
||||
|
||||
// StopAsking ends the request to GeoJS under way, and returns once the
|
||||
// goroutine that made it has. A request it ends is no failure: Failed is not
|
||||
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
||||
// asked nothing.
|
||||
func (g *GeoJS) StopAsking() {
|
||||
g.mu.Lock()
|
||||
|
||||
g.stopped = true
|
||||
if g.asking {
|
||||
g.endAsking()
|
||||
}
|
||||
|
||||
g.mu.Unlock()
|
||||
|
||||
g.askGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||
g.mu.Lock()
|
||||
@@ -294,7 +316,8 @@ func (g *GeoJS) answerOrWait(
|
||||
}
|
||||
|
||||
if !g.asking {
|
||||
// GeoJS is left alone after a failure, so no answer can come.
|
||||
// GeoJS is left alone after a failure, or asked nothing after
|
||||
// StopAsking, so no answer can come.
|
||||
w.late = true
|
||||
}
|
||||
|
||||
@@ -323,22 +346,28 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||
}
|
||||
|
||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||
// it is under way or it is left alone after a failure. The requests to
|
||||
// GeoJS are for every client waiting, so they go on when the caller whose
|
||||
// ctx is given stops waiting.
|
||||
// it is under way, it is left alone after a failure, or StopAsking has
|
||||
// been called. The requests to GeoJS are for every client waiting, so they
|
||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
||||
func (g *GeoJS) ask(ctx context.Context) {
|
||||
if g.asking || g.now().Before(g.retryAt) {
|
||||
if g.asking || g.stopped || g.now().Before(g.retryAt) {
|
||||
return
|
||||
}
|
||||
|
||||
g.asking = true
|
||||
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
g.asking, g.endAsking = true, end
|
||||
|
||||
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
||||
g.askGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
g.askAboutWaiting(askCtx)
|
||||
})
|
||||
}
|
||||
|
||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||
// time, until none is left or GeoJS fails. Each answer kept is given to
|
||||
// Answered, outside the lock, since Answered takes locks of its own.
|
||||
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
||||
// answer kept is given to Answered, outside the lock, since Answered takes
|
||||
// locks of its own.
|
||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
for {
|
||||
clients := g.nextClients()
|
||||
@@ -347,6 +376,10 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
}
|
||||
|
||||
given, err := g.request(ctx, clients)
|
||||
if err != nil && ctx.Err() != nil {
|
||||
continue // ended by StopAsking, which is no failure
|
||||
}
|
||||
|
||||
kept, answered := g.keep(clients, given, err)
|
||||
|
||||
if g.answered != nil {
|
||||
@@ -362,13 +395,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
}
|
||||
|
||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||
// none is waiting, it returns none and notes that no request to GeoJS is
|
||||
// under way.
|
||||
// none is waiting, or StopAsking has been called, it returns none and
|
||||
// notes that no request to GeoJS is under way.
|
||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
if len(g.waiting) == 0 {
|
||||
if len(g.waiting) == 0 || g.stopped {
|
||||
g.asking = false
|
||||
|
||||
return nil
|
||||
|
||||
@@ -318,6 +318,44 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
service, _, g, reports := startWithReports()
|
||||
clients := newClients()
|
||||
|
||||
service.set(hanging)
|
||||
g.LookUpWithoutWaiting(t.Context(), clients())
|
||||
waitForRequests(t, service, 1)
|
||||
|
||||
// The request is ended, not waited out, and is no failure.
|
||||
began := time.Now()
|
||||
|
||||
g.StopAsking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
reports.wantFailures(t)
|
||||
|
||||
// A new client is then not asked about, and its lookup does not
|
||||
// wait.
|
||||
service.set(answering)
|
||||
wantCountry(t, g, clients(), "")
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("the lookup returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
waitForRequests(t, service, 1)
|
||||
reports.wantFailures(t)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+18
-49
@@ -13,18 +13,13 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/oschwald/maxminddb-golang/v2"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||
)
|
||||
|
||||
// quietTime is how long the lookup database must go without a change
|
||||
// before it is read again, so that a file still being copied in is read
|
||||
// only once whole.
|
||||
const quietTime = 2 * time.Second
|
||||
|
||||
// FileParams are what OpenFile needs.
|
||||
type FileParams struct {
|
||||
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
||||
@@ -122,20 +117,13 @@ func (f *File) ReadFailures() int {
|
||||
}
|
||||
|
||||
// Watch watches the directory of the lookup database until ctx is done,
|
||||
// and reads the file again once it has gone without a change for
|
||||
// quietTime, after it is replaced, written or removed, and after Watch
|
||||
// starts watching. If the directory cannot be watched, that is logged, and
|
||||
// the database read at start stays in use.
|
||||
// and reads the file again once it has gone without a change for a short
|
||||
// time, after it is replaced, written or removed, and after Watch starts
|
||||
// watching, as dirwatch describes. A change to another file in its
|
||||
// directory does not count. If the directory cannot be watched, that is
|
||||
// logged, and the database read at start stays in use.
|
||||
func (f *File) Watch(ctx context.Context) {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err == nil {
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
err = watcher.Add(filepath.Dir(f.params.Path))
|
||||
}
|
||||
|
||||
watcher, err := dirwatch.New(filepath.Dir(f.params.Path))
|
||||
if err != nil {
|
||||
f.params.ProcessLog.Error(
|
||||
"cannot watch the lookup database for replacements",
|
||||
@@ -146,41 +134,22 @@ func (f *File) Watch(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
||||
"file", f.params.Path)
|
||||
|
||||
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
||||
watcher.CallWhenQuiet(ctx, f.isDatabase, f.readAgain, func(err error) {
|
||||
f.params.ProcessLog.Warn("watching the lookup database failed",
|
||||
"error", err.Error())
|
||||
})
|
||||
}
|
||||
|
||||
// readAfterChanges reads the lookup database again once quietTime has
|
||||
// passed without a change to it from events, until ctx is done, and logs
|
||||
// the errors from errs. A change to another file in its directory does not
|
||||
// count. The wait starts at once, as if for a change, so that a file
|
||||
// replaced after OpenFile read it, and before its directory was watched,
|
||||
// is read too.
|
||||
func (f *File) readAfterChanges(
|
||||
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
||||
) {
|
||||
path := filepath.Clean(f.params.Path)
|
||||
|
||||
quiet := time.NewTimer(quietTime)
|
||||
defer quiet.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case event := <-events:
|
||||
if filepath.Clean(event.Name) == path {
|
||||
quiet.Reset(quietTime)
|
||||
}
|
||||
case <-quiet.C:
|
||||
f.readAgain()
|
||||
case err := <-errs:
|
||||
f.params.ProcessLog.Warn("watching the lookup database failed",
|
||||
"error", err.Error())
|
||||
}
|
||||
}
|
||||
// isDatabase reports whether path names the lookup database.
|
||||
func (f *File) isDatabase(path string) bool {
|
||||
return filepath.Clean(path) == filepath.Clean(f.params.Path)
|
||||
}
|
||||
|
||||
// readAgain reads the lookup database again, in place of the one in use,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package lookup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
@@ -12,7 +11,6 @@ import (
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/maxmind/mmdbwriter/mmdbtype"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
@@ -133,66 +131,48 @@ func TestFileThatCannotBeReadIsAnError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The tests below run readAfterChanges in a synctest bubble, where time is
|
||||
// a clock of the test's own: time.Sleep moves it on at once, and
|
||||
// synctest.Wait returns once readAfterChanges waits again, so that every
|
||||
// reading due by then is done. The test sends the changes itself, as the
|
||||
// watch of a directory cannot run in a bubble.
|
||||
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
func TestReplacementCopiedOverTheFileInTwoPartsIsReadOnlyWhole(
|
||||
t *testing.T,
|
||||
) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||
writeDatabase(t, path, "DE")
|
||||
|
||||
f := openFile(t, path, newQueue())
|
||||
|
||||
for changed, want := range map[string]bool{
|
||||
path: true,
|
||||
dir + "/./ipinfo_lite.mmdb": true,
|
||||
filepath.Join(dir, "other.mmdb"): false,
|
||||
} {
|
||||
if f.isDatabase(changed) != want {
|
||||
t.Errorf(
|
||||
"a change to %s counts: %t, want %t",
|
||||
changed,
|
||||
!want,
|
||||
want,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The tests below run in a synctest bubble, where time is a clock of the
|
||||
// test's own, so that the time a file is read can be checked. The wait for
|
||||
// the directory to go quiet before a reading is tested in dirwatch.
|
||||
|
||||
func TestReplacementIsReadAgain(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||
writeDatabase(t, path, "DE")
|
||||
|
||||
queue := newQueue()
|
||||
f := openFile(t, path, queue)
|
||||
changes := watch(t, f)
|
||||
|
||||
other := filepath.Join(dir, "replacement.mmdb")
|
||||
writeDatabase(t, other, "KP")
|
||||
|
||||
//nolint:gosec // a file the test wrote
|
||||
replacement, err := os.ReadFile(other)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", other, err)
|
||||
}
|
||||
|
||||
// The file in use is overwritten in place, and keeps giving what
|
||||
// it gave. Its first part alone is not a .mmdb file.
|
||||
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
||||
if err != nil {
|
||||
t.Fatalf("create %s: %v", path, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
half := len(replacement) / 2
|
||||
write(t, file, replacement[:half])
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "DE")
|
||||
|
||||
// The second part starts the wait again.
|
||||
write(t, file, replacement[half:])
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "DE")
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
writeDatabase(t, path, "KP")
|
||||
time.Sleep(time.Hour)
|
||||
f.readAgain()
|
||||
wantCountry(t, f, "KP")
|
||||
|
||||
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
||||
@@ -219,15 +199,10 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
||||
queue := newQueue()
|
||||
f := openFile(t, path, queue)
|
||||
read := f.LastRead()
|
||||
changes := watch(t, f)
|
||||
|
||||
writeFile(t, path, "not a lookup database\n")
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
// Long after, the replacement has been read once.
|
||||
time.Sleep(time.Hour)
|
||||
synctest.Wait()
|
||||
f.readAgain()
|
||||
wantCountry(t, f, "DE")
|
||||
|
||||
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
||||
@@ -240,7 +215,7 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
||||
}
|
||||
|
||||
wantAlerts(t, queue, alerts.Alert{
|
||||
Time: read.Add(quietTime),
|
||||
Time: read.Add(time.Hour),
|
||||
Event: alerts.EventFileError,
|
||||
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
||||
Detail: map[string]any{
|
||||
@@ -252,55 +227,6 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
||||
})
|
||||
}
|
||||
|
||||
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||
writeDatabase(t, path, "DE")
|
||||
|
||||
f := openFile(t, path, newQueue())
|
||||
changes := watch(t, f)
|
||||
|
||||
// The wait that starts with the watch ends with a reading.
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
writeDatabase(t, path, "KP")
|
||||
|
||||
changes <- fsnotify.Event{Name: filepath.Join(dir, "other.mmdb"), Op: fsnotify.Create}
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "DE")
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "KP")
|
||||
})
|
||||
}
|
||||
|
||||
func TestReplacementSavedBeforeTheWatchStartsIsRead(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||
writeDatabase(t, path, "DE")
|
||||
|
||||
f := openFile(t, path, newQueue())
|
||||
|
||||
// Saved after OpenFile read the file, and before its directory was
|
||||
// watched, so that no change is seen for it.
|
||||
writeDatabase(t, path, "KP")
|
||||
watch(t, f)
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
wantCountry(t, f, "KP")
|
||||
})
|
||||
}
|
||||
|
||||
// newQueue returns a queue of alerts for a webhook that is never sent
|
||||
// them, so that they wait in it for the test to look at.
|
||||
func newQueue() *alerts.Queue {
|
||||
@@ -344,28 +270,6 @@ func openFile(t *testing.T, path string, queue *alerts.Queue) *File {
|
||||
return f
|
||||
}
|
||||
|
||||
// watch runs f's readAfterChanges until the test ends, and returns the
|
||||
// channel that sends it changes.
|
||||
func watch(t *testing.T, f *File) chan<- fsnotify.Event {
|
||||
t.Helper()
|
||||
|
||||
changes := make(chan fsnotify.Event)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
f.readAfterChanges(ctx, changes, nil)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
<-stopped
|
||||
})
|
||||
|
||||
return changes
|
||||
}
|
||||
|
||||
// wantCountry checks the country f gives testClient.
|
||||
func wantCountry(t *testing.T, f *File, want string) {
|
||||
t.Helper()
|
||||
@@ -401,13 +305,3 @@ func writeFile(t *testing.T, path, content string) {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// write writes data to the end of file.
|
||||
func write(t *testing.T, file *os.File, data []byte) {
|
||||
t.Helper()
|
||||
|
||||
_, err := file.Write(data)
|
||||
if err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
@@ -301,7 +300,7 @@ const sourceLabel = "source"
|
||||
|
||||
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||
// config.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
||||
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
||||
// the fetches that failed and when the copy in use was fetched, and for a
|
||||
@@ -320,7 +319,9 @@ func (m *Metrics) AddReputation(
|
||||
m.registry.MustRegister(m.reputationHits)
|
||||
|
||||
for _, zone := range dnsbl.Zones() {
|
||||
source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
|
||||
source := prometheus.Labels{
|
||||
sourceLabel: reputation.MaskZoneKey(zone),
|
||||
}
|
||||
|
||||
m.addReputationQueries(
|
||||
source,
|
||||
|
||||
@@ -110,8 +110,12 @@ type AbuseIPDB struct {
|
||||
// scores are by client. Each is added as it is fetched and never moved
|
||||
// up, so that the one fetched longest ago is the first dropped.
|
||||
scores *simplelru.LRU[netip.Prefix, Score]
|
||||
// checking are the clients whose check is under way.
|
||||
checking map[netip.Prefix]bool
|
||||
// checking are the clients whose check is under way, each with the
|
||||
// function that ends it. stopped is true once StopChecking has been
|
||||
// called, and checkGoroutines waits for the goroutine of each check.
|
||||
checking map[netip.Prefix]context.CancelFunc
|
||||
stopped bool
|
||||
checkGoroutines sync.WaitGroup
|
||||
// day is the day, in UTC, of the checks spent counts.
|
||||
day civil.Date
|
||||
spent int
|
||||
@@ -134,7 +138,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
||||
params: params,
|
||||
httpClient: &http.Client{},
|
||||
scores: scores,
|
||||
checking: map[netip.Prefix]bool{},
|
||||
checking: map[netip.Prefix]context.CancelFunc{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,11 +148,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
||||
// addresses its request comes from. A client without one is checked in
|
||||
// the background, by addr, the address its request came from, if
|
||||
// offender, if it has committed an offence, unless its check is under
|
||||
// way, a check failed less than failureDelay ago, or the day's checks
|
||||
// have used up DailyBudget; Hit never waits for a check. The check that
|
||||
// uses the budget up is logged and raised as a source_failure alert. ctx
|
||||
// is the context of the client's request, and a check goes on after the
|
||||
// request ends.
|
||||
// way, a check failed less than failureDelay ago, the day's checks have
|
||||
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
||||
// for a check. The check that uses the budget up is logged and raised as a
|
||||
// source_failure alert. ctx is the context of the client's request, and a
|
||||
// check goes on after the request ends, until StopChecking.
|
||||
func (a *AbuseIPDB) Hit(
|
||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||
) (int64, bool) {
|
||||
@@ -167,14 +171,21 @@ func (a *AbuseIPDB) Hit(
|
||||
a.day, a.spent = today, 0
|
||||
}
|
||||
|
||||
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
||||
a.spent < a.params.DailyBudget
|
||||
_, checking := a.checking[client]
|
||||
|
||||
check := offender && !checking && !a.stopped &&
|
||||
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
||||
if check {
|
||||
a.checking[client] = true
|
||||
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
a.checking[client] = end
|
||||
a.checks++
|
||||
a.spent++
|
||||
|
||||
go a.check(context.WithoutCancel(ctx), client, addr)
|
||||
a.checkGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
a.check(checkCtx, client, addr)
|
||||
})
|
||||
}
|
||||
|
||||
usedUp := check && a.spent == a.params.DailyBudget
|
||||
@@ -189,6 +200,24 @@ func (a *AbuseIPDB) Hit(
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// StopChecking ends the checks under way, and returns once their goroutines
|
||||
// have. A check it ends gives no score and is no failure: it is neither
|
||||
// counted, logged nor raised as an alert, and checks are not left alone
|
||||
// after it. The ended check stays spent from the day's budget. After
|
||||
// StopChecking, Hit checks no client.
|
||||
func (a *AbuseIPDB) StopChecking() {
|
||||
a.mu.Lock()
|
||||
|
||||
a.stopped = true
|
||||
for _, end := range a.checking {
|
||||
end()
|
||||
}
|
||||
|
||||
a.mu.Unlock()
|
||||
|
||||
a.checkGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Checked returns how many checks were made.
|
||||
func (a *AbuseIPDB) Checked() int {
|
||||
a.mu.Lock()
|
||||
@@ -315,7 +344,8 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||
// the score as client's, and notes the check as no longer under way. A
|
||||
// check that fails gives no score: it is counted, logged and raised as a
|
||||
// source_failure alert, and no client is checked for failureDelay.
|
||||
// source_failure alert, and no client is checked for failureDelay. A check
|
||||
// ended as ctx ends, by StopChecking, is no failure.
|
||||
func (a *AbuseIPDB) check(
|
||||
ctx context.Context,
|
||||
client netip.Prefix,
|
||||
@@ -323,24 +353,26 @@ func (a *AbuseIPDB) check(
|
||||
) {
|
||||
score, err := a.ask(ctx, addr)
|
||||
now := a.params.Now()
|
||||
isFailure := err != nil && ctx.Err() == nil
|
||||
|
||||
a.mu.Lock()
|
||||
|
||||
delete(a.checking, client)
|
||||
|
||||
if err == nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
a.scores.Add(
|
||||
client,
|
||||
Score{Client: client, Score: score, Fetched: now},
|
||||
)
|
||||
} else {
|
||||
case isFailure:
|
||||
a.failures++
|
||||
a.retryAt = now.Add(failureDelay)
|
||||
}
|
||||
|
||||
a.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if isFailure {
|
||||
a.alert("checking a client with AbuseIPDB failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -394,6 +394,43 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
||||
})
|
||||
}
|
||||
|
||||
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
queue := newQueue()
|
||||
p := abuseIPDBParams()
|
||||
p.Alerts = queue
|
||||
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
||||
checker := newAbuseIPDB(abuseIPDB, p)
|
||||
|
||||
wantScore(t, checker, suspect, true, 0, false)
|
||||
synctest.Wait()
|
||||
|
||||
// The check is ended, not waited out, and is no failure.
|
||||
began := time.Now()
|
||||
|
||||
checker.StopChecking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopChecking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
wantFailures(t, checker, 0)
|
||||
|
||||
if got := waiting(queue); len(got) != 0 {
|
||||
t.Errorf("alerts waiting %+v, want none", got)
|
||||
}
|
||||
|
||||
// An offender without a score is then not checked.
|
||||
wantScore(t, checker, other, true, 0, false)
|
||||
synctest.Wait()
|
||||
wantChecked(t, abuseIPDB, suspect)
|
||||
})
|
||||
}
|
||||
|
||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
|
||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -32,6 +31,8 @@ const (
|
||||
// it fails, and no client is checked with AbuseIPDB after a check
|
||||
// fails, so that a source refusing them is not asked on every request.
|
||||
failureDelay = time.Minute
|
||||
// maskedKey is what MaskZoneKey shows in place of a zone's key.
|
||||
maskedKey = "********"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -82,8 +83,12 @@ type DNSBL struct {
|
||||
// verdicts are by query. Each is added as it is fetched and never moved
|
||||
// up, so that the one fetched longest ago is the first dropped.
|
||||
verdicts *simplelru.LRU[query, Verdict]
|
||||
// asking are the queries under way.
|
||||
asking map[query]bool
|
||||
// asking are the queries under way, each with the function that ends
|
||||
// it. stopped is true once StopAsking has been called, and
|
||||
// queryGoroutines waits for the goroutine of each query.
|
||||
asking map[query]context.CancelFunc
|
||||
stopped bool
|
||||
queryGoroutines sync.WaitGroup
|
||||
// queries and failures count, by zone, the queries made and those that
|
||||
// failed, and retryAt is when a zone whose last query failed may be
|
||||
// asked again.
|
||||
@@ -120,13 +125,31 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
||||
params: params,
|
||||
resolver: resolver,
|
||||
verdicts: verdicts,
|
||||
asking: map[query]bool{},
|
||||
asking: map[query]context.CancelFunc{},
|
||||
queries: map[string]int{},
|
||||
failures: map[string]int{},
|
||||
retryAt: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
||||
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
||||
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
||||
// other zone it returns as it is. A zone is shown so wherever it leaves
|
||||
// the process: in the log, the alerts and the metrics.
|
||||
func MaskZoneKey(zone string) string {
|
||||
// DNS names ignore case, and a name may be written with a dot at its
|
||||
// end.
|
||||
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
||||
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
||||
return zone
|
||||
}
|
||||
|
||||
_, rest, _ := strings.Cut(zone, ".")
|
||||
|
||||
return maskedKey + "." + rest
|
||||
}
|
||||
|
||||
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
||||
func (d *DNSBL) Zones() []string {
|
||||
return slices.Clone(d.params.Zones)
|
||||
@@ -134,13 +157,14 @@ func (d *DNSBL) Zones() []string {
|
||||
|
||||
// ListedBy returns the zones whose verdict on addr, a client's address,
|
||||
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
||||
// masked, as config.MaskZoneKey masks it, since they go to the request
|
||||
// masked, as MaskZoneKey masks it, since they go to the request
|
||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||
// passed since it was fetched. Each zone without one is asked about addr
|
||||
// in the background, unless a query about addr to it is under way, the
|
||||
// zone is left alone after a failure, or maxQueries are under way;
|
||||
// ListedBy never waits for a query. ctx is the context of the client's
|
||||
// request, and a query goes on after the request ends.
|
||||
// zone is left alone after a failure, maxQueries are under way, or
|
||||
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
||||
// context of the client's request, and a query goes on after the request
|
||||
// ends, until StopAsking.
|
||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
@@ -153,23 +177,47 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||
q := query{zone: zone, client: addr}
|
||||
|
||||
kept, found := d.verdicts.Peek(q)
|
||||
_, asking := d.asking[q]
|
||||
|
||||
switch {
|
||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||
if kept.Listed {
|
||||
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
||||
listedBy = append(listedBy, MaskZoneKey(zone))
|
||||
}
|
||||
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
||||
d.asking[q] = true
|
||||
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
||||
len(d.asking) < maxQueries:
|
||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
d.asking[q] = end
|
||||
d.queries[zone]++
|
||||
|
||||
go d.ask(context.WithoutCancel(ctx), q)
|
||||
d.queryGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
d.ask(queryCtx, q)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return listedBy
|
||||
}
|
||||
|
||||
// StopAsking ends the queries under way, and returns once their
|
||||
// goroutines have. A query it ends gives no verdict and is no failure: it
|
||||
// is neither counted, logged nor raised as an alert, and the zone is not
|
||||
// left alone after it. After StopAsking, ListedBy asks no zone.
|
||||
func (d *DNSBL) StopAsking() {
|
||||
d.mu.Lock()
|
||||
|
||||
d.stopped = true
|
||||
for _, end := range d.asking {
|
||||
end()
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
d.queryGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Queries returns how many queries were made to zone.
|
||||
func (d *DNSBL) Queries(zone string) int {
|
||||
d.mu.Lock()
|
||||
@@ -238,30 +286,32 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
||||
// query as no longer under way. A query that fails gives no verdict: it
|
||||
// is counted, logged and raised as a source_failure alert, which show the
|
||||
// zone with its key masked, and the zone is not asked again for
|
||||
// failureDelay.
|
||||
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||
listed, err := d.lookUp(ctx, q)
|
||||
now := d.params.Now()
|
||||
isFailure := err != nil && ctx.Err() == nil
|
||||
|
||||
d.mu.Lock()
|
||||
|
||||
delete(d.asking, q)
|
||||
|
||||
if err == nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
d.verdicts.Add(q, Verdict{
|
||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||
})
|
||||
} else {
|
||||
case isFailure:
|
||||
d.failures[q.zone]++
|
||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if isFailure {
|
||||
const failed = "asking a DNSBL zone failed"
|
||||
|
||||
shown := config.MaskZoneKey(q.zone)
|
||||
shown := MaskZoneKey(q.zone)
|
||||
|
||||
// Raised before it is logged, so that the alert is there once the
|
||||
// log line is.
|
||||
|
||||
@@ -210,6 +210,45 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
//
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
queue := newQueue()
|
||||
p := dnsblParams(zone)
|
||||
p.Alerts = queue
|
||||
resolver := &resolverStandIn{hanging: true}
|
||||
dnsbl := newDNSBL(resolver, p)
|
||||
|
||||
wantZones(t, dnsbl, listed)
|
||||
synctest.Wait()
|
||||
|
||||
// The query is ended, not waited out, and is no failure.
|
||||
began := time.Now()
|
||||
|
||||
dnsbl.StopAsking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
wantQueries(t, dnsbl, 1, 0)
|
||||
|
||||
if got := waiting(queue); len(got) != 0 {
|
||||
t.Errorf("alerts waiting %+v, want none", got)
|
||||
}
|
||||
|
||||
// A client without a verdict is then asked about in no zone.
|
||||
wantZones(t, dnsbl, unlisted)
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 1, 0)
|
||||
wantAsked(t, resolver, listedName)
|
||||
waitForTheResolver()
|
||||
})
|
||||
}
|
||||
|
||||
//
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||
@@ -821,3 +860,28 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
|
||||
t.Errorf("asked about %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
dronebl = "dnsbl.dronebl.org"
|
||||
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
|
||||
spamhausMasked = "********.xbl.dq.spamhaus.net"
|
||||
)
|
||||
|
||||
for zone, want := range map[string]string{
|
||||
spamhaus: spamhausMasked,
|
||||
spamhaus + ".": spamhausMasked + ".",
|
||||
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
||||
dronebl: dronebl,
|
||||
"dq.spamhaus.net": "dq.spamhaus.net",
|
||||
spamhaus + ".example": spamhaus + ".example",
|
||||
} {
|
||||
if got := reputation.MaskZoneKey(zone); got != want {
|
||||
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,8 +7,10 @@
|
||||
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
||||
// today. The state package writes all of these to files and reads them
|
||||
// from there, so that a restart keeps them too. Last, it fetches
|
||||
// AbuseIPDB's blacklist, which it does not keep.
|
||||
// from there, so that a restart keeps them too. It fetches AbuseIPDB's
|
||||
// blacklist, which it does not keep. Last, it reads the netblocks, AS
|
||||
// numbers and percentages that both the settings and the lists hold, and
|
||||
// masks the keys of DNSBL zones.
|
||||
package reputation
|
||||
|
||||
import (
|
||||
@@ -20,12 +22,12 @@ import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -40,15 +42,28 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
errStatus = errors.New("the server answered")
|
||||
errTooLong = errors.New("the list is longer than 16 MiB")
|
||||
errNotNetblock = errors.New(
|
||||
errStatus = errors.New("the server answered")
|
||||
errTooLong = errors.New("the list is longer than 16 MiB")
|
||||
errNotNetblockLine = errors.New(
|
||||
"is not an address or a netblock, such as 192.0.2.0/24",
|
||||
)
|
||||
errNotASNPercent = errors.New(
|
||||
"is not an AS number, : and a percentage, such as AS64496:50")
|
||||
errNotNetblock = errors.New(
|
||||
"is not a netblock such as 10.0.0.0/8, or an address")
|
||||
errNotASN = errors.New("is not an AS number such as AS64496")
|
||||
errNotPercent = errors.New(
|
||||
"is not a percentage, a whole number from 0 to 100",
|
||||
)
|
||||
)
|
||||
|
||||
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
||||
// requests of the clients it holds, and the URL it is fetched from.
|
||||
type TagList struct {
|
||||
Name string
|
||||
URL string
|
||||
}
|
||||
|
||||
// List is a list as lists.json holds it: the URL it is fetched from,
|
||||
// when it was last tried, the fetch failed or not, and its last good copy:
|
||||
// when that was fetched, and its lines, as fetched, comment lines
|
||||
@@ -68,7 +83,7 @@ type Params struct {
|
||||
BlocklistURLs []string
|
||||
ASNLimitPercentURL string
|
||||
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
||||
TagLists []config.TagList
|
||||
TagLists []TagList
|
||||
// Refresh is how long after a list was last fetched or tried it is
|
||||
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
||||
Refresh time.Duration
|
||||
@@ -465,7 +480,11 @@ func parseNetblocks(lines []string) (entries, error) {
|
||||
|
||||
netblock, ok := parseNetblock(text)
|
||||
if !ok {
|
||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
|
||||
return entries{}, fmt.Errorf(
|
||||
"line %d %w",
|
||||
i+1,
|
||||
errNotNetblockLine,
|
||||
)
|
||||
}
|
||||
|
||||
found.netblocks[netblock] = true
|
||||
@@ -485,7 +504,7 @@ func parseNetblocks(lines []string) (entries, error) {
|
||||
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
||||
// stands for none, is refused.
|
||||
func parseNetblock(text string) (netip.Prefix, bool) {
|
||||
netblock, err := config.ParseNetblock(text)
|
||||
netblock, err := ParseNetblock(text)
|
||||
if err != nil {
|
||||
return netip.Prefix{}, false
|
||||
}
|
||||
@@ -520,9 +539,9 @@ func parsePercents(lines []string) (entries, error) {
|
||||
}
|
||||
|
||||
asnText, percentText, _ := strings.Cut(text, ":")
|
||||
asn, asnErr := config.ParseASN(asnText)
|
||||
asn, asnErr := ParseASN(asnText)
|
||||
|
||||
percent, percentErr := config.ParsePercent(percentText)
|
||||
percent, percentErr := ParsePercent(percentText)
|
||||
if asnErr != nil || percentErr != nil {
|
||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
||||
}
|
||||
@@ -536,6 +555,53 @@ func parsePercents(lines []string) (entries, error) {
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
||||
// address is a netblock of that address alone, a /32 or a /128. A
|
||||
// blocklist's lines are read with it too.
|
||||
func ParseNetblock(value string) (netip.Prefix, error) {
|
||||
if strings.Contains(value, "/") {
|
||||
netblock, err := netip.ParsePrefix(value)
|
||||
if err != nil {
|
||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||
}
|
||||
|
||||
return netblock.Masked(), nil
|
||||
}
|
||||
|
||||
addr, err := netip.ParseAddr(value)
|
||||
if err != nil || addr.Zone() != "" {
|
||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||
}
|
||||
|
||||
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
||||
}
|
||||
|
||||
// ParseASN reads an AS number such as AS64496, in either case, and
|
||||
// returns it as the lookup gives it: AS and the number, in capitals and
|
||||
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
||||
// read with it too.
|
||||
func ParseASN(value string) (string, error) {
|
||||
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
||||
|
||||
number, err := strconv.ParseUint(digits, 10, 32)
|
||||
if !hasAS || err != nil {
|
||||
return "", fmt.Errorf("%q %w", value, errNotASN)
|
||||
}
|
||||
|
||||
return "AS" + strconv.FormatUint(number, 10), nil
|
||||
}
|
||||
|
||||
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
||||
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
||||
func ParsePercent(value string) (int64, error) {
|
||||
percent, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || percent < 0 || percent > 100 {
|
||||
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
||||
}
|
||||
|
||||
return percent, nil
|
||||
}
|
||||
|
||||
// withoutComment returns line without anything after a ; or a #, and
|
||||
// without the spaces around what is left.
|
||||
func withoutComment(line string) string {
|
||||
|
||||
@@ -99,8 +99,12 @@ type Client struct {
|
||||
// answers are by client. Each is added as it comes and never moved up,
|
||||
// so that the one that came longest ago is the first dropped.
|
||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||
// asking are the clients whose query is under way.
|
||||
asking map[netip.Prefix]bool
|
||||
// asking are the clients whose query is under way, each with the
|
||||
// function that ends it. stopped is true once StopAsking has been
|
||||
// called, and queryGoroutines waits for the goroutine of each query.
|
||||
asking map[netip.Prefix]context.CancelFunc
|
||||
stopped bool
|
||||
queryGoroutines sync.WaitGroup
|
||||
// retryAt is when routewatch may be asked again after a query failed.
|
||||
retryAt time.Time
|
||||
}
|
||||
@@ -116,7 +120,7 @@ func New(params Params) *Client {
|
||||
params: params,
|
||||
httpClient: &http.Client{},
|
||||
answers: answers,
|
||||
asking: map[netip.Prefix]bool{},
|
||||
asking: map[netip.Prefix]context.CancelFunc{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,8 +128,9 @@ func New(params Params) *Client {
|
||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||
// without an answer is asked about in the background, by addr, the address its
|
||||
// request came from, unless its query is under way, a query failed less than
|
||||
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
||||
// client's request, and a query goes on after the request ends.
|
||||
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
||||
// is the context of the client's request, and a query goes on after the request
|
||||
// ends, until StopAsking.
|
||||
func (c *Client) Route(
|
||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||
) (Route, Missing) {
|
||||
@@ -147,15 +152,38 @@ func (c *Client) Route(
|
||||
return Route{}, RequestFailed
|
||||
}
|
||||
|
||||
if !c.asking[client] && len(c.asking) < maxQueries {
|
||||
c.asking[client] = true
|
||||
_, asking := c.asking[client]
|
||||
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
c.asking[client] = end
|
||||
|
||||
go c.ask(context.WithoutCancel(ctx), client, addr)
|
||||
c.queryGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
c.ask(queryCtx, client, addr)
|
||||
})
|
||||
}
|
||||
|
||||
return Route{}, NoAnswerYet
|
||||
}
|
||||
|
||||
// StopAsking ends the queries under way, and returns once their goroutines
|
||||
// have. A query it ends gives no answer and is no failure: it is not logged,
|
||||
// and routewatch is not left alone after it. After StopAsking, Route asks
|
||||
// routewatch nothing.
|
||||
func (c *Client) StopAsking() {
|
||||
c.mu.Lock()
|
||||
|
||||
c.stopped = true
|
||||
for _, end := range c.asking {
|
||||
end()
|
||||
}
|
||||
|
||||
c.mu.Unlock()
|
||||
|
||||
c.queryGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Snapshot returns every answer still in use, sorted by client, as
|
||||
// routewatch.json lists them.
|
||||
func (c *Client) Snapshot() []Answer {
|
||||
@@ -205,7 +233,8 @@ func (c *Client) Load(answers []Answer) {
|
||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||
// query as no longer under way. A query that fails gives no answer, and
|
||||
// routewatch is not asked again for failureKept. Only the first failure of
|
||||
// those under way is logged.
|
||||
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
||||
// failure.
|
||||
func (c *Client) ask(
|
||||
ctx context.Context,
|
||||
client netip.Prefix,
|
||||
@@ -213,18 +242,20 @@ func (c *Client) ask(
|
||||
) {
|
||||
route, err := c.query(ctx, addr)
|
||||
now := c.params.Now()
|
||||
isFailure := err != nil && ctx.Err() == nil
|
||||
|
||||
c.mu.Lock()
|
||||
|
||||
delete(c.asking, client)
|
||||
|
||||
firstFailure := err != nil && !now.Before(c.retryAt)
|
||||
firstFailure := isFailure && !now.Before(c.retryAt)
|
||||
|
||||
if err == nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
c.answers.Add(client, Answer{
|
||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||
})
|
||||
} else {
|
||||
case isFailure:
|
||||
c.retryAt = now.Add(failureKept)
|
||||
}
|
||||
|
||||
|
||||
@@ -361,6 +361,54 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
||||
// says.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
f := &fixture{t: t, log: &syncBuffer{}}
|
||||
f.client = routewatch.New(routewatch.Params{
|
||||
URL: silentURL, Now: time.Now,
|
||||
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
||||
})
|
||||
|
||||
f.route()
|
||||
synctest.Wait()
|
||||
|
||||
// The query is ended, not waited out.
|
||||
began := time.Now()
|
||||
|
||||
f.client.StopAsking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
// A query started now would fail within waitLimit. None is, and the
|
||||
// one ended is no failure: nothing is logged, and routewatch is not
|
||||
// left alone.
|
||||
for range 2 {
|
||||
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
||||
t.Errorf(
|
||||
"Route gave %q, want %q",
|
||||
missing,
|
||||
routewatch.NoAnswerYet,
|
||||
)
|
||||
}
|
||||
|
||||
time.Sleep(waitLimit)
|
||||
synctest.Wait()
|
||||
}
|
||||
|
||||
if f.log.String() != "" {
|
||||
t.Errorf("process log %q, want nothing", f.log.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -2,6 +2,12 @@ package rproxy
|
||||
|
||||
import "time"
|
||||
|
||||
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
|
||||
// long requests take cannot matter to it. It is called before s serves.
|
||||
func (s *Server) StopBreakerClock(at time.Time) {
|
||||
s.h.breakerClock = func() time.Time { return at }
|
||||
}
|
||||
|
||||
// Breaker is the circuit breaker, for tests that give it the time.
|
||||
type Breaker struct {
|
||||
b breaker
|
||||
|
||||
@@ -94,6 +94,7 @@ func New(params Params) *Server {
|
||||
reportURL: (&url.URL{
|
||||
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
||||
}).String(),
|
||||
breakerClock: time.Now,
|
||||
}
|
||||
|
||||
return &Server{
|
||||
@@ -133,8 +134,10 @@ type handler struct {
|
||||
wafTransport http.RoundTripper
|
||||
// reportURL is where the reports go.
|
||||
reportURL string
|
||||
// breaker is the circuit breaker for asks to waf.
|
||||
breaker breaker
|
||||
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
|
||||
// the time.
|
||||
breaker breaker
|
||||
breakerClock func() time.Time
|
||||
// reports waits for the reports being sent. reportsMu guards starting one
|
||||
// against reportsStopped, which is set once nothing is to wait for another.
|
||||
reports sync.WaitGroup
|
||||
|
||||
@@ -356,6 +356,19 @@ func startServerLogging(
|
||||
) (string, *rproxy.Server) {
|
||||
t.Helper()
|
||||
|
||||
server := newServer(t, backend, appURL, env, requestLog, processLog)
|
||||
|
||||
return serve(t, server), server
|
||||
}
|
||||
|
||||
// newServer returns rproxy, not yet serving, with its request lines going to
|
||||
// requestLog and its process lines to processLog.
|
||||
func newServer(
|
||||
t *testing.T, backend, appURL string, env map[string]string,
|
||||
requestLog, processLog io.Writer,
|
||||
) *rproxy.Server {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||
wafBackendTimeout: "60000",
|
||||
@@ -367,7 +380,7 @@ func startServerLogging(
|
||||
t.Fatalf("settings %v: %v", settings, err)
|
||||
}
|
||||
|
||||
server := rproxy.New(rproxy.Params{
|
||||
return rproxy.New(rproxy.Params{
|
||||
Config: cfg,
|
||||
RequestLog: requestLog,
|
||||
ProcessLog: requestlog.NewProcessLogger(
|
||||
@@ -376,6 +389,11 @@ func startServerLogging(
|
||||
slog.LevelInfo,
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
// serve has server serve until the test ends, and returns its address.
|
||||
func serve(t *testing.T, server *rproxy.Server) string {
|
||||
t.Helper()
|
||||
|
||||
listener, err := (&net.ListenConfig{}).Listen(
|
||||
t.Context(),
|
||||
@@ -396,7 +414,7 @@ func startServerLogging(
|
||||
server.WaitForReports()
|
||||
})
|
||||
|
||||
return listener.Addr().String(), server
|
||||
return listener.Addr().String()
|
||||
}
|
||||
|
||||
// answered is the visitor's answer.
|
||||
@@ -1532,9 +1550,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
|
||||
tooLarge(w, r)
|
||||
})
|
||||
app := startApp(t, greet)
|
||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||
wafBackendTimeout: "60000",
|
||||
})
|
||||
out := &output{}
|
||||
server := newServer(t, standIn.URL, app.URL, nil, out, out)
|
||||
// The breaker's clock stands still. On a slow host, the five failures could
|
||||
// otherwise take more than the five seconds that open the breaker. The next
|
||||
// request could come more than a second later, when it may ask waf again.
|
||||
server.StopBreakerClock(time.Now())
|
||||
addr := serve(t, server)
|
||||
|
||||
for range 5 {
|
||||
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
@@ -97,7 +96,7 @@ func (rq *request) askUnlessBreakerOpen(
|
||||
ctx context.Context,
|
||||
) (*wafAnswer, error) {
|
||||
b := &rq.h.breaker
|
||||
if !b.mayAsk(time.Now()) {
|
||||
if !b.mayAsk(rq.h.breakerClock()) {
|
||||
return nil, errBreakerOpen
|
||||
}
|
||||
|
||||
@@ -107,7 +106,7 @@ func (rq *request) askUnlessBreakerOpen(
|
||||
case err == nil:
|
||||
b.noteAnswer()
|
||||
case ctx.Err() == nil:
|
||||
b.noteFailure(time.Now())
|
||||
b.noteFailure(rq.h.breakerClock())
|
||||
}
|
||||
|
||||
return answer, err
|
||||
|
||||
+21
-50
@@ -18,12 +18,11 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"golang.org/x/net/http/httpguts"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
||||
)
|
||||
|
||||
// The actions a rule takes when it matches.
|
||||
@@ -40,12 +39,6 @@ const (
|
||||
// extension ends the name of every rule file.
|
||||
const extension = ".rules"
|
||||
|
||||
// quietTime is how long SWW_WAF_RULE_FILES_DIR must go without a change before
|
||||
// the rule files are read again, so that a file still being written, such
|
||||
// as one saved in place, appended to or copied in with scp, is read only
|
||||
// once whole.
|
||||
const quietTime = 2 * time.Second
|
||||
|
||||
// headerTarget starts the target that is one request header,
|
||||
// header:<Name>.
|
||||
const headerTarget = "header:"
|
||||
@@ -176,26 +169,18 @@ func (f *Files) Len() int {
|
||||
}
|
||||
|
||||
// Watch watches Dir until ctx is done, and reads the rule files again
|
||||
// once Dir has had no change for quietTime, after one is edited, added or
|
||||
// removed, and after Watch starts watching. If they then hold an error,
|
||||
// the rules stay as they were, the error is logged with its file and
|
||||
// line, and the files are read again after the next change. If Dir cannot
|
||||
// be watched, that is logged, and the rules stay as they were loaded.
|
||||
// While Enabled is false, Watch returns at once.
|
||||
// once Dir has had no change for a short time, after one is edited, added
|
||||
// or removed, and after Watch starts watching, as dirwatch describes. If
|
||||
// they then hold an error, the rules stay as they were, the error is
|
||||
// logged with its file and line, and the files are read again after the
|
||||
// next change. If Dir cannot be watched, that is logged, and the rules
|
||||
// stay as they were loaded. While Enabled is false, Watch returns at once.
|
||||
func (f *Files) Watch(ctx context.Context) {
|
||||
if !f.params.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err == nil {
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
err = watcher.Add(f.params.Dir)
|
||||
}
|
||||
|
||||
watcher, err := dirwatch.New(f.params.Dir)
|
||||
if err != nil {
|
||||
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
||||
"error", err.Error())
|
||||
@@ -203,36 +188,22 @@ func (f *Files) Watch(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = watcher.Close()
|
||||
}()
|
||||
|
||||
f.params.ProcessLog.Info("watching the rule files for edits",
|
||||
"directory", f.params.Dir)
|
||||
|
||||
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
||||
}
|
||||
|
||||
// readAfterChanges reads the rule files again once quietTime has passed
|
||||
// without a change from events, until ctx is done, and logs the errors
|
||||
// from errs. The wait starts at once, as if for a change, so that an edit
|
||||
// saved after Load read the files, and before Dir was watched, is taken
|
||||
// in too.
|
||||
func (f *Files) readAfterChanges(
|
||||
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
||||
) {
|
||||
quiet := time.NewTimer(quietTime)
|
||||
defer quiet.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-events:
|
||||
quiet.Reset(quietTime)
|
||||
case <-quiet.C:
|
||||
f.readAgain()
|
||||
case err := <-errs:
|
||||
// A change to any file in Dir counts.
|
||||
watcher.CallWhenQuiet(ctx,
|
||||
func(string) bool { return true },
|
||||
f.readAgain,
|
||||
func(err error) {
|
||||
f.params.ProcessLog.Warn("watching the rule files failed",
|
||||
"error", err.Error())
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// readAgain reads the rule files again, in place of the rules loaded, or
|
||||
@@ -378,7 +349,7 @@ func parse(line string) (Rule, bool, error) {
|
||||
rule.Target,
|
||||
errNotTarget,
|
||||
)
|
||||
case isHeader && !config.IsHeaderName(headerName):
|
||||
case isHeader && !httpguts.ValidHeaderFieldName(headerName):
|
||||
return Rule{}, false, fmt.Errorf(
|
||||
"the target %q %w",
|
||||
rule.Target,
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
package rules
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
)
|
||||
|
||||
// The tests below run readAfterChanges in a synctest bubble, where time is
|
||||
// a clock of the test's own: time.Sleep moves it on at once, and
|
||||
// synctest.Wait returns once readAfterChanges waits again, so that every
|
||||
// reading due by then is done. The test sends the changes itself, as the
|
||||
// watch of a directory cannot run in a bubble.
|
||||
|
||||
func TestFileWrittenInTwoPartsTakenInOnlyWhole(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "50-app.rules")
|
||||
writeFile(t, path, "first path block ^/first\n")
|
||||
files := load(t, dir)
|
||||
changes := run(t, files)
|
||||
|
||||
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
// The first part ends in the middle of a ban rule's regex, which,
|
||||
// read then, would ban every request.
|
||||
write(t, file, "first path block ^/first\nprobe path ban ^/")
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantMatched(t, files, "/anything")
|
||||
|
||||
// The second part starts the wait again.
|
||||
write(t, file, `\.env$`+"\n")
|
||||
|
||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||
|
||||
time.Sleep(quietTime - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantMatched(t, files, "/.env")
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantMatched(t, files, "/.env", "probe")
|
||||
wantMatched(t, files, "/anything")
|
||||
})
|
||||
}
|
||||
|
||||
func TestEditSavedBeforeTheWatchStartsTakenIn(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "50-app.rules")
|
||||
writeFile(t, path, "first path block ^/first\n")
|
||||
files := load(t, dir)
|
||||
|
||||
// Saved after Load read the files, and before the directory was
|
||||
// watched, so that no change is seen for it.
|
||||
writeFile(t, path, "first path block ^/edited\n")
|
||||
run(t, files)
|
||||
time.Sleep(quietTime)
|
||||
synctest.Wait()
|
||||
wantMatched(t, files, "/edited", "first")
|
||||
})
|
||||
}
|
||||
|
||||
// load loads the rules in dir.
|
||||
func load(t *testing.T, dir string) *Files {
|
||||
t.Helper()
|
||||
|
||||
files, err := Load(Params{
|
||||
Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler),
|
||||
Alerts: alerts.New(alerts.Params{}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// run runs files' readAfterChanges until the test ends, and returns the
|
||||
// channel that sends it changes.
|
||||
func run(t *testing.T, files *Files) chan<- fsnotify.Event {
|
||||
t.Helper()
|
||||
|
||||
changes := make(chan fsnotify.Event)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
files.readAfterChanges(ctx, changes, nil)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
<-stopped
|
||||
})
|
||||
|
||||
return changes
|
||||
}
|
||||
|
||||
// writeFile writes content to the file at path.
|
||||
func writeFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
|
||||
err := os.WriteFile(path, []byte(content), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// write writes text to the end of file.
|
||||
func write(t *testing.T, file *os.File, text string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := file.WriteString(text)
|
||||
if err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// wantMatched checks the ids of the rules that a GET request for path
|
||||
// matches, in order.
|
||||
func wantMatched(t *testing.T, files *Files, path string, want ...string) {
|
||||
t.Helper()
|
||||
|
||||
r := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
"http://app.example"+path, nil)
|
||||
|
||||
matched := files.Match(r)
|
||||
|
||||
got := make([]string, 0, len(matched))
|
||||
for _, rule := range matched {
|
||||
got = append(got, rule.ID)
|
||||
}
|
||||
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("%s matched %v, want %v", path, got, want)
|
||||
}
|
||||
}
|
||||
@@ -755,8 +755,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
|
||||
// read from it ends with an error once waitLimit is up.
|
||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
|
||||
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||
t.Helper()
|
||||
|
||||
@@ -769,11 +768,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||
_ = conn.Close()
|
||||
})
|
||||
|
||||
err = conn.SetDeadline(time.Now().Add(waitLimit))
|
||||
if err != nil {
|
||||
t.Fatalf("set the deadline: %v", err)
|
||||
}
|
||||
|
||||
_, err = io.WriteString(conn, request)
|
||||
if err != nil {
|
||||
t.Fatalf("send: %v", err)
|
||||
@@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
||||
|
||||
stop()
|
||||
|
||||
// Counted from the stop, so a slow wait for the app cannot shorten it.
|
||||
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
|
||||
|
||||
text, err := io.ReadAll(conn)
|
||||
if err != nil || len(text) != 0 {
|
||||
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
||||
|
||||
@@ -660,7 +660,9 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
|
||||
|
||||
status := run(t.Context(), map[string]string{
|
||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
||||
"SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
|
||||
"SWW_WAF_CORE_RULE_SET_MODE": string(
|
||||
config.CoreRuleSetModeOff,
|
||||
),
|
||||
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
||||
}, out)
|
||||
if status != 1 {
|
||||
|
||||
+65
-16
@@ -2,6 +2,7 @@ package sww
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
@@ -117,7 +118,8 @@ func RunWAF(ctx context.Context, params Params) int {
|
||||
"settings", cfg)
|
||||
|
||||
// The last write follows the stop of every hook, and so the end of every
|
||||
// loop that changes the state or raises an alert. A hook of the state
|
||||
// loop that changes the state or raises an alert, and of every ask of
|
||||
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
||||
// files would stop before the loops of the parts built ahead of them.
|
||||
return run(ctx, app, failed, processLog, func() error {
|
||||
err := files.WriteAll()
|
||||
@@ -175,11 +177,35 @@ func newRuleFiles(
|
||||
return ruleFiles, nil
|
||||
}
|
||||
|
||||
// newCoreRuleSet returns the Core Rule Set, as waf.NewCoreRuleSet does.
|
||||
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
||||
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
||||
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
||||
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
||||
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
||||
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
||||
// set file is loaded with the Core Rule Set only to check it.
|
||||
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||
crs, err := waf.NewCoreRuleSet(cfg)
|
||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
||||
cfg.CoreRuleSetExclusionsFile == "" {
|
||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||
}
|
||||
|
||||
crs, err := coreruleset.New(coreruleset.Params{
|
||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
||||
Apps: cfg.Apps,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, startError{"cannot use the Core Rule Set", err}
|
||||
return nil, startError{
|
||||
"cannot use the Core Rule Set",
|
||||
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||
}
|
||||
|
||||
return crs, nil
|
||||
@@ -192,7 +218,7 @@ func newLookupFile(
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
alertQueue *alerts.Queue,
|
||||
) (*lookup.File, error) {
|
||||
if cfg.LookupSource != "file" {
|
||||
if cfg.LookupSource != config.LookupSourceFile {
|
||||
return nil, nil //nolint:nilnil // no lookup database, and no error
|
||||
}
|
||||
|
||||
@@ -272,11 +298,13 @@ func newLists(
|
||||
return lists
|
||||
}
|
||||
|
||||
// newDNSBL returns what asks the DNSBL zones, none asked yet.
|
||||
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
||||
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
||||
func newDNSBL(
|
||||
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
alertQueue *alerts.Queue,
|
||||
) *reputation.DNSBL {
|
||||
return reputation.NewDNSBL(reputation.DNSBLParams{
|
||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
||||
Zones: cfg.DNSBLZones,
|
||||
Resolver: cfg.DNSBLResolver,
|
||||
CacheTTL: cfg.IPReputationCacheTTL,
|
||||
@@ -285,12 +313,17 @@ func newDNSBL(
|
||||
ProcessLog: processLog,
|
||||
Alerts: alertQueue,
|
||||
})
|
||||
|
||||
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
||||
|
||||
return dnsbl
|
||||
}
|
||||
|
||||
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
||||
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
||||
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
||||
// ledger, as waf.BanAbuseIPDBBlacklist says.
|
||||
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
||||
// under way, as reputation.AbuseIPDB.StopChecking says.
|
||||
func newAbuseIPDB(
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
||||
@@ -316,24 +349,30 @@ func newAbuseIPDB(
|
||||
})
|
||||
|
||||
lc.Append(background(abuseIPDB.RunBlacklist))
|
||||
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
||||
|
||||
return abuseIPDB
|
||||
}
|
||||
|
||||
// newRouteWatch returns what asks routewatch about clients, or nil while
|
||||
// SWW_WAF_ROUTEWATCH_URL is unset.
|
||||
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
||||
// way, as routewatch.Client.StopAsking says.
|
||||
func newRouteWatch(
|
||||
cfg *config.WAF, processLog *slog.Logger,
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
) *routewatch.Client {
|
||||
if cfg.RouteWatchURL == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return routewatch.New(routewatch.Params{
|
||||
routeWatch := routewatch.New(routewatch.Params{
|
||||
URL: cfg.RouteWatchURL,
|
||||
Now: utcNow,
|
||||
ProcessLog: processLog,
|
||||
})
|
||||
|
||||
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
||||
|
||||
return routeWatch
|
||||
}
|
||||
|
||||
// newMetrics returns the metrics, with those read from the other parts as
|
||||
@@ -371,12 +410,18 @@ func newMetrics(
|
||||
|
||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
||||
// failure is handled as waf.GeoJSFailed says.
|
||||
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
||||
// under way, as geojs.GeoJS.StopAsking says.
|
||||
func newGeoJS(
|
||||
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
|
||||
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
|
||||
lc fx.Lifecycle,
|
||||
cfg *config.WAF,
|
||||
m *metrics.Metrics,
|
||||
limiter *ratelimit.Limiter,
|
||||
ledger *bans.Ledger,
|
||||
alertQueue *alerts.Queue,
|
||||
processLog *slog.Logger,
|
||||
) *geojs.GeoJS {
|
||||
return geojs.New(geojs.Params{
|
||||
geoJS := geojs.New(geojs.Params{
|
||||
URL: geojs.URL,
|
||||
Answered: func(answer geojs.Answer) {
|
||||
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||
@@ -388,6 +433,10 @@ func newGeoJS(
|
||||
},
|
||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||
})
|
||||
|
||||
lc.Append(fx.StopHook(geoJS.StopAsking))
|
||||
|
||||
return geoJS
|
||||
}
|
||||
|
||||
// newWAFServer returns the server waf runs, with its parts and the time
|
||||
|
||||
@@ -54,7 +54,7 @@ func (abuseIPDB) check(
|
||||
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
||||
)
|
||||
|
||||
if rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -86,13 +86,14 @@ func (rq *request) countedBytes() int64 {
|
||||
response, request := rq.out.bytes, rq.requestBytes
|
||||
|
||||
switch rq.h.config.BytesCounted {
|
||||
case "response":
|
||||
case config.BytesCountedResponse:
|
||||
return response
|
||||
case "request":
|
||||
case config.BytesCountedRequest:
|
||||
return request
|
||||
default: // both
|
||||
return response + request
|
||||
case config.BytesCountedBoth:
|
||||
}
|
||||
|
||||
return response + request
|
||||
}
|
||||
|
||||
// banForLimit bans the client's netblock at now for a broken limit, the
|
||||
|
||||
+10
-9
@@ -53,33 +53,34 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
if country == "" && !rq.skippedLookupWait {
|
||||
unknown = percentage{
|
||||
cfg.UnknownCountryLimitPercent,
|
||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
||||
config.UnknownCountryLimitPercentSetting,
|
||||
}
|
||||
}
|
||||
|
||||
fetched := percentage{percent: whole}
|
||||
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
||||
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
|
||||
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
|
||||
}
|
||||
|
||||
blocklisted := percentage{percent: whole}
|
||||
if rq.blocklisted && cfg.BlocklistAction == "limit" {
|
||||
if rq.blocklisted &&
|
||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
||||
blocklisted = percentage{
|
||||
cfg.BlocklistLimitPercent,
|
||||
"SWW_WAF_BLOCKLIST_ACTION",
|
||||
config.BlocklistActionSetting,
|
||||
}
|
||||
}
|
||||
|
||||
dnsblAbuseIPDBListed := percentage{percent: whole}
|
||||
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
||||
cfg.DNSBLAbuseIPDBAction == "limit" {
|
||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
||||
dnsblAbuseIPDBListed = percentage{
|
||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
||||
config.DNSBLAbuseIPDBActionSetting,
|
||||
}
|
||||
}
|
||||
|
||||
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||
const tagSetting = config.TagLimitPercentSetting
|
||||
|
||||
asnRequests := lowest(
|
||||
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
||||
@@ -107,7 +108,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
asnBytes = given(
|
||||
cfg.ASNBytesLimitPercent,
|
||||
asn,
|
||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
||||
config.ASNBytesLimitPercentSetting,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -115,7 +116,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
countryBytes = given(
|
||||
cfg.CountryBytesLimitPercent,
|
||||
country,
|
||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
||||
config.CountryBytesLimitPercentSetting,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
@@ -41,5 +42,6 @@ func (blocklists) check(
|
||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
||||
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
|
||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
|
||||
config.ReputationActionDeny
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func (coreRuleSet) check(
|
||||
rq.line.CoreRuleSetScore = &result.Score
|
||||
|
||||
for _, id := range result.RuleIDs {
|
||||
rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
|
||||
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
|
||||
}
|
||||
|
||||
threshold := cfg.CoreRuleSetScoreThreshold
|
||||
@@ -99,7 +99,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
||||
|
||||
switch {
|
||||
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
||||
detail["mode"] = config.CoreRuleSetModeDetect
|
||||
detail["mode"] = string(config.CoreRuleSetModeDetect)
|
||||
case rq.h.config.Observe:
|
||||
detail["mode"] = "observe"
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
@@ -33,7 +34,8 @@ func (countries) check(
|
||||
return &verdict{
|
||||
action: requestlog.ActionCountryDenied,
|
||||
figures: map[string]any{
|
||||
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||
"country": country,
|
||||
"setting": config.ExclusivelyAllowedCountriesSetting,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,8 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
||||
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
||||
}
|
||||
|
||||
if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||
if !rq.dnsblListed ||
|
||||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -104,12 +104,6 @@ func onReport() []heuristic {
|
||||
}
|
||||
}
|
||||
|
||||
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
|
||||
// it is called.
|
||||
func (rq *request) runAfterAnswer(ctx context.Context) {
|
||||
rq.runBanningHeuristics(ctx, afterAnswer())
|
||||
}
|
||||
|
||||
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
||||
// and returns the first verdict, or nil when none gives one. The log line's
|
||||
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
||||
|
||||
@@ -49,7 +49,8 @@ func (rq *request) startLookUp(
|
||||
) <-chan lookupResult {
|
||||
result := make(chan lookupResult, 1)
|
||||
|
||||
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
|
||||
if rq.h.config.LookupSource == config.LookupSourceOff ||
|
||||
!canBePlaced(rq.client) {
|
||||
result <- lookupResult{}
|
||||
|
||||
return result
|
||||
@@ -57,7 +58,7 @@ func (rq *request) startLookUp(
|
||||
|
||||
client := clientGroup(rq.h.config, rq.client)
|
||||
|
||||
if rq.h.config.LookupSource == "file" {
|
||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
||||
result <- lookupResult{
|
||||
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
|
||||
}
|
||||
|
||||
+2
-37
@@ -6,7 +6,6 @@ package waf
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -83,8 +82,8 @@ type Params struct {
|
||||
// LookupFile is the lookup database they are looked up in while
|
||||
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
||||
LookupFile *lookup.File
|
||||
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
|
||||
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
||||
// CoreRuleSet is what inspects requests while
|
||||
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
||||
CoreRuleSet CoreRuleSetInspector
|
||||
// Now tells the time by which requests are counted for the rate
|
||||
// limits, bans are made and run out, and GeoJS's answers are kept,
|
||||
@@ -174,40 +173,6 @@ func newHandler(params Params) *handler {
|
||||
}
|
||||
}
|
||||
|
||||
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
||||
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
||||
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
||||
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
||||
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
||||
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
||||
// set file is loaded with the Core Rule Set only to check it.
|
||||
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
||||
cfg.CoreRuleSetExclusionsFile == "" {
|
||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||
}
|
||||
|
||||
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
||||
Apps: cfg.Apps,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||
}
|
||||
|
||||
return coreRuleSet, nil
|
||||
}
|
||||
|
||||
// handler is waf's handler. It holds what every request shares; what
|
||||
// belongs to one request is in a request.
|
||||
type handler struct {
|
||||
|
||||
+34
-13
@@ -20,6 +20,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
@@ -504,24 +505,20 @@ func newWAFWith(
|
||||
}
|
||||
}
|
||||
|
||||
coreRuleSet, err := waf.NewCoreRuleSet(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Core Rule Set: %v", err)
|
||||
}
|
||||
|
||||
params := waf.Params{
|
||||
Config: cfg,
|
||||
RequestLog: out,
|
||||
ProcessLog: processLog,
|
||||
LookupFile: lookupFile,
|
||||
CoreRuleSet: coreRuleSet,
|
||||
Now: now,
|
||||
Rules: ruleFiles,
|
||||
Alerts: alertQueue,
|
||||
Config: cfg,
|
||||
RequestLog: out,
|
||||
ProcessLog: processLog,
|
||||
LookupFile: lookupFile,
|
||||
Now: now,
|
||||
Rules: ruleFiles,
|
||||
Alerts: alertQueue,
|
||||
// The tests of the limit set it back, and hold the Core Rule Set
|
||||
// with a stand-in.
|
||||
BesideLookupTimeLimit: time.Hour,
|
||||
}
|
||||
|
||||
addCoreRuleSet(t, ¶ms)
|
||||
addCounters(¶ms)
|
||||
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
||||
|
||||
@@ -530,6 +527,30 @@ func newWAFWith(
|
||||
return server, out, alertQueue
|
||||
}
|
||||
|
||||
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
|
||||
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
|
||||
func addCoreRuleSet(t *testing.T, params *waf.Params) {
|
||||
t.Helper()
|
||||
|
||||
cfg := params.Config
|
||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
||||
return
|
||||
}
|
||||
|
||||
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
||||
Apps: cfg.Apps,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Core Rule Set: %v", err)
|
||||
}
|
||||
|
||||
params.CoreRuleSet = coreRuleSet
|
||||
}
|
||||
|
||||
// addCounters adds to params the metrics, the table of clients, the ledger
|
||||
// and the anomaly counters, as sww builds them from params' settings. The
|
||||
// metrics read the ledger, the table of clients and the rules.
|
||||
|
||||
@@ -7,10 +7,6 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
)
|
||||
|
||||
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
|
||||
// refuses the requests of a client a source lists.
|
||||
const deny = "deny"
|
||||
|
||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
||||
// with reason.
|
||||
|
||||
+11
-1
@@ -345,6 +345,16 @@ func timing(start, end time.Time) *float64 {
|
||||
return new(requestlog.Milliseconds(end.Sub(start)))
|
||||
}
|
||||
|
||||
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
|
||||
// It then counts the request for the anomaly thresholds and adds it to its
|
||||
// client's history. list runs first, so that a refusal limit it finds broken
|
||||
// is in that history. ctx is the request's own context.
|
||||
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
|
||||
rq.runBanningHeuristics(ctx, list)
|
||||
rq.countAnomalies()
|
||||
rq.addToHistory()
|
||||
}
|
||||
|
||||
// addToHistory adds the request, which has ended, to its client's
|
||||
// history, and counts its offences in the metrics, and then the lookup's
|
||||
// answer about the client, as answerAtTheEnd gives it, to that history and
|
||||
@@ -421,7 +431,7 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
|
||||
return geojs.Answer{}, false
|
||||
}
|
||||
|
||||
if rq.h.config.LookupSource == "file" {
|
||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
||||
return rq.lookupAnswer, true
|
||||
}
|
||||
|
||||
|
||||
+21
-28
@@ -177,11 +177,13 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
||||
peer.Unmap(),
|
||||
true,
|
||||
)
|
||||
// The last deferred runs first, so that what is held has the finished log
|
||||
// line. Go's server sends the answer only once this function has
|
||||
// returned, so the report cannot come before it is held.
|
||||
defer rq.holdForReport()
|
||||
defer rq.finish()
|
||||
// These run even when a check panics. finish runs first, so that what is
|
||||
// held has the finished log line. Go's server sends the answer only once
|
||||
// this function has returned, so the report cannot come before it is held.
|
||||
defer func() {
|
||||
rq.finish()
|
||||
rq.holdForReport()
|
||||
}()
|
||||
|
||||
rq.site = r.Host
|
||||
rq.line.WAFLine = requestlog.WAFLineDecision
|
||||
@@ -190,35 +192,28 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
||||
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
|
||||
|
||||
if refused == nil && !underOwnPrefix {
|
||||
// In observe mode, what enforce mode would have refused counts here.
|
||||
defer rq.runAfterAnswer(r.Context())
|
||||
|
||||
rq.allow()
|
||||
// In observe mode, what enforce mode would have refused counts here.
|
||||
rq.runBanningHeuristics(r.Context(), afterAnswer())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// The last deferred runs first: runAfterAnswer before addToHistory, so that
|
||||
// a broken refusal limit is in the client's history.
|
||||
defer rq.addToHistory()
|
||||
defer rq.countAnomalies()
|
||||
defer rq.runAfterAnswer(r.Context())
|
||||
|
||||
if refused != nil {
|
||||
rq.deny(*refused)
|
||||
|
||||
return
|
||||
} else {
|
||||
// The app never sees this prefix. waf's own endpoints are at its
|
||||
// own address, so this answers as a path with no endpoint does.
|
||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
||||
http.Error(
|
||||
rq.out,
|
||||
http.StatusText(http.StatusNotFound),
|
||||
http.StatusNotFound,
|
||||
)
|
||||
}
|
||||
|
||||
// The app never sees this prefix. waf's own endpoints are at its
|
||||
// own address, so this answers as a path with no endpoint does.
|
||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
||||
http.Error(
|
||||
rq.out,
|
||||
http.StatusText(http.StatusNotFound),
|
||||
http.StatusNotFound,
|
||||
)
|
||||
rq.banAndCount(r.Context(), afterAnswer())
|
||||
}
|
||||
|
||||
// readQuestion returns the visitor's address and the body a question carries,
|
||||
@@ -491,9 +486,7 @@ func (h *handler) finishWith(
|
||||
if rq.allowed {
|
||||
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
||||
|
||||
rq.runBanningHeuristics(ctx, onReport())
|
||||
rq.countAnomalies()
|
||||
rq.addToHistory()
|
||||
rq.banAndCount(ctx, onReport())
|
||||
|
||||
rq.h.metrics.RequestReported(
|
||||
&rq.line,
|
||||
|
||||
Reference in new issue
Block a user