package sww import ( "context" "io" "log/slog" "net" "net/http" "time" "go.uber.org/fx" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/coreruleset" "sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/routewatch" "sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/state" "sneak.berlin/go/smallwebwaf/internal/waf" ) // remoteLogStopTimeout is how long, as waf stops, the log lines still // waiting are sent to SWW_WAF_LOG_REMOTE_URL before they are given up. stdout has // carried them. const remoteLogStopTimeout = 2 * time.Second // RunWAF is `sww waf`: it reads the settings, the rule files, // the lookup database and the state and cache files, then serves requests until // ctx is done. Then it gives the requests in progress shutdownTimeout to // finish, ends its background work and writes every state and cache file, // alerts.json with the alerts still waiting. It returns the process's exit // status, 1 when waf cannot start or stop in order. func RunWAF(ctx context.Context, params Params) int { // Until the settings are read, the one message is an invalid setting's // error, which every SWW_WAF_LOG_LEVEL lets through. processLog := requestlog.NewProcessLogger(params.Stdout, config.InstanceName(params.Viper), slog.LevelError) cfg, err := config.WAFFromEnvironment(params.Viper) if err != nil { return cannotStart(processLog, startError{"invalid setting", err}) } // While SWW_WAF_LOG_REMOTE_URL is set, every line on stdout from here on // is sent there too. The sender runs before any part is built, so that it // sends why waf cannot start. stdout := params.Stdout var remote *remotelog.Sender if cfg.LogRemoteURL != nil { remote = newRemoteLogSender(cfg) stdout = io.MultiWriter(params.Stdout, remote) } processLog = requestlog.NewProcessLogger( stdout, cfg.InstanceName, cfg.LogLevel, ) if remote != nil { stopSending := startSending(ctx, remote, processLog) defer stopSending() } var ( files *state.Files listener net.Listener ) failed := make(servingFailed, 1) app := fx.New( fx.Supply( params, cfg, processLog, requestLog{stdout}, remote, failed, ), fx.Provide( newAlertQueue, newRuleFiles, newCoreRuleSet, newLookupFile, newLimiter, newLedger, newAnomalyCounters, newLists, newDNSBL, newAbuseIPDB, newRouteWatch, newMetrics, newGeoJS, newWAFServer, func(server *waf.Server) *http.Server { return server.Server }, newWAFListener, newStateFiles, ), // fx stops hooks in the reverse of the order they were added in. The // state files are read before serving is set up, so that serving // stops before their loops do. fx.Populate(&files), fx.Invoke(serve), fx.Populate(&listener), fx.WithLogger(fxEventLogger(processLog)), ) err = app.Err() if err != nil { return cannotStart(processLog, err) } processLog.Info("starting", "version", params.Version, "address", listener.Addr().String(), "settings", cfg) // The last write follows the stop of every hook, and so the end of every // loop that changes the state or raises an alert. A hook of the state // files would stop before the loops of the parts built ahead of them. return run(ctx, app, failed, processLog, func() error { err := files.WriteAll() if err != nil { processLog.Error("writing the state and cache files failed", "error", err.Error()) } return err }) } // newAlertQueue returns the queue of the alerts to the webhook, Slack and // ntfy, with the settings for them. Until waf stops, it sends them. func newAlertQueue( lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger, ) *alerts.Queue { alertQueue := alerts.New(alerts.Params{ WebhookURL: cfg.AlertWebhookURL, WebhookHeaders: cfg.AlertWebhookHeaders, SlackURL: cfg.AlertSlackWebhookURL, NtfyURL: cfg.AlertNtfyURL, NtfyToken: cfg.AlertNtfyToken, Events: cfg.AlertEvents, Cooldown: cfg.AlertCooldown, MaxPerHour: cfg.AlertMaxPerHour, Instance: cfg.InstanceName, Now: utcNow, ProcessLog: processLog, }) lc.Append(background(alertQueue.Run)) return alertQueue } // newRuleFiles reads the rule files, and reads them again as they change // until waf stops. func newRuleFiles( lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue, ) (*rules.Files, error) { ruleFiles, err := rules.Load(rules.Params{ Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog, Alerts: alertQueue, }) if err != nil { return nil, startError{"cannot use the rule files", err} } lc.Append(background(ruleFiles.Watch)) return ruleFiles, nil } // newCoreRuleSet returns the Core Rule Set, as waf.NewCoreRuleSet does. func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) { crs, err := waf.NewCoreRuleSet(cfg) if err != nil { return nil, startError{"cannot use the Core Rule Set", err} } return crs, nil } // newLookupFile reads the lookup database while SWW_WAF_LOOKUP_SOURCE is // file, and reads it again when it is replaced until waf stops. It returns // nil while the source is anything else. func newLookupFile( lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue, ) (*lookup.File, error) { if cfg.LookupSource != "file" { return nil, nil //nolint:nilnil // no lookup database, and no error } lookupFile, err := lookup.OpenFile(lookup.FileParams{ Path: cfg.LookupDBPath, Now: utcNow, ProcessLog: processLog, Alerts: alertQueue, }) if err != nil { return nil, startError{"cannot use the lookup database", err} } lc.Append(background(lookupFile.Watch)) return lookupFile, nil } // newLimiter returns the table of clients, with the rate and byte limits. func newLimiter(cfg *config.WAF) *ratelimit.Limiter { return ratelimit.New(ratelimit.Limits{ PerMinute: cfg.RateLimitPerMinute, PerHour: cfg.RateLimitPerHour, PerDay: cfg.RateLimitPerDay, BytesPerMinute: cfg.BytesLimitPerMinute, BytesPerHour: cfg.BytesLimitPerHour, BytesPerDay: cfg.BytesLimitPerDay, }, cfg.MaxTrackedClients) } // newLedger returns the ledger of the bans, with the settings for them. func newLedger(cfg *config.WAF) *bans.Ledger { return bans.New(bans.Params{ LimitBanDuration: cfg.LimitBanDuration, LimitBanRepeatWindow: cfg.LimitBanRepeatWindow, LimitBanPermanentAbove: cfg.LimitBanPermanentAbove, AttackBanDuration: cfg.AttackBanDuration, MaxBans: cfg.MaxBans, }) } // newAnomalyCounters returns the counters of the anomaly thresholds. func newAnomalyCounters( cfg *config.WAF, alertQueue *alerts.Queue, ) *anomaly.Counters { return anomaly.New(anomaly.Params{ Client: cfg.AnomalyClient, Net: cfg.AnomalyNet, ASN: cfg.AnomalyASN, Total: cfg.AnomalyTotal, WatchedNet: cfg.AnomalyWatchedNet, NetV4Prefix: cfg.AnomalyNetV4Prefix, NetV6Prefix: cfg.AnomalyNetV6Prefix, NamedNetblocks: cfg.AnomalyWatchedNets, Alerts: alertQueue, }) } // newLists returns the lists the settings name by URL, none fetched yet. // Until waf stops, it fetches them as they are due. func newLists( lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue, ) *reputation.Lists { lists := reputation.New(reputation.Params{ BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh, ASNLimitPercentURL: cfg.ASNLimitPercentURL, TagLists: cfg.TagLists, Now: utcNow, ProcessLog: processLog, Alerts: alertQueue, }) lc.Append(background(lists.Run)) return lists } // newDNSBL returns what asks the DNSBL zones, none asked yet. func newDNSBL( cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue, ) *reputation.DNSBL { return reputation.NewDNSBL(reputation.DNSBLParams{ Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver, CacheTTL: cfg.IPReputationCacheTTL, Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: utcNow, ProcessLog: processLog, Alerts: alertQueue, }) } // newAbuseIPDB returns what checks clients with AbuseIPDB while // SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. func newAbuseIPDB( cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue, ) *reputation.AbuseIPDB { return reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{ URL: reputation.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore, DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.IPReputationCacheTTL, Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: utcNow, ProcessLog: processLog, Alerts: alertQueue, }) } // newRouteWatch returns what asks routewatch about clients, or nil while // SWW_WAF_ROUTEWATCH_URL is unset. func newRouteWatch( cfg *config.WAF, processLog *slog.Logger, ) *routewatch.Client { if cfg.RouteWatchURL == "" { return nil } return routewatch.New(routewatch.Params{ URL: cfg.RouteWatchURL, Now: utcNow, ProcessLog: processLog, }) } // newMetrics returns the metrics, with those read from the other parts as // the metrics are asked for: AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set, // the lookup database's while there is one, and the remote log sender's // while there is one. func newMetrics( cfg *config.WAF, ledger *bans.Ledger, limiter *ratelimit.Limiter, ruleFiles *rules.Files, lists *reputation.Lists, dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB, alertQueue *alerts.Queue, lookupFile *lookup.File, remote *remotelog.Sender, ) *metrics.Metrics { m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName) m.AddBansAndClients(ledger, limiter, utcNow) m.AddRules(ruleFiles) m.AddReputation(lists, dnsbl) if cfg.AbuseIPDBKey != "" { m.AddAbuseIPDB(abuseIPDB) } m.AddAlerts(alertQueue) if lookupFile != nil { m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures) } if remote != nil { m.AddRemoteLog(remote) } return m } // newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each // answer goes to the limiter and the ledger, as waf.AddLookup says, and each // failure is handled as waf.GeoJSFailed says. func newGeoJS( cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter, ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger, ) *geojs.GeoJS { return geojs.New(geojs.Params{ URL: geojs.URL, Answered: func(answer geojs.Answer) { waf.AddLookup(cfg, limiter, ledger, answer) }, Now: utcNow, Asked: m.GeoJSRequests.Inc, Failed: func(err error, askingAgainIn time.Duration) { waf.GeoJSFailed(processLog, alertQueue, m, err, askingAgainIn) }, WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc, }) } // newWAFServer returns the server waf runs, with its parts and the time // limit params gives. func newWAFServer( params Params, cfg *config.WAF, out requestLog, processLog *slog.Logger, m *metrics.Metrics, limiter *ratelimit.Limiter, ledger *bans.Ledger, anomalies *anomaly.Counters, lists *reputation.Lists, dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB, routeWatch *routewatch.Client, geoJS *geojs.GeoJS, lookupFile *lookup.File, crs *coreruleset.CoreRuleSet, ruleFiles *rules.Files, alertQueue *alerts.Queue, ) *waf.Server { return waf.New(waf.Params{ Config: cfg, RequestLog: out, ProcessLog: processLog, Metrics: m, Limiter: limiter, Ledger: ledger, Anomalies: anomalies, Lists: lists, DNSBL: dnsbl, AbuseIPDB: abuseIPDB, RouteWatch: routeWatch, GeoJS: geoJS, LookupFile: lookupFile, CoreRuleSet: crs, Now: utcNow, Rules: ruleFiles, Alerts: alertQueue, BesideLookupTimeLimit: params.BesideLookupTimeLimit, }) } // newWAFListener listens on SWW_WAF_LISTEN_ADDR. func newWAFListener(cfg *config.WAF) (net.Listener, error) { return listen(cfg.ListenAddr, "SWW_WAF_LISTEN_ADDR") } // newStateFiles reads the state and cache files into the parts and the // alert queue, as state.Load does. Until waf stops, it writes them as they // are due and takes in an admin's edits of them. func newStateFiles( lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger, m *metrics.Metrics, ledger *bans.Ledger, limiter *ratelimit.Limiter, geoJS *geojs.GeoJS, lists *reputation.Lists, dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB, routeWatch *routewatch.Client, alertQueue *alerts.Queue, anomalies *anomaly.Counters, ) (*state.Files, error) { files, err := state.Load(state.Params{ Dir: cfg.StateDir, CacheDir: cfg.CacheDir, BansWriteDelay: cfg.BansWriteDelay, WriteInterval: cfg.StateWriteInterval, Ledger: ledger, Limiter: limiter, GeoJS: geoJS, Lists: lists, DNSBL: dnsbl, AbuseIPDB: abuseIPDB, RouteWatch: routeWatch, Alerts: alertQueue, Anomalies: anomalies, Now: utcNow, ProcessLog: processLog, Metrics: m, }) if err != nil { return nil, startError{"cannot use the state and cache files", err} } lc.Append(background(files.Run)) lc.Append(background(files.Watch)) return files, nil } // newRemoteLogSender returns a sender of the log lines to // SWW_WAF_LOG_REMOTE_URL, with the settings for it. func newRemoteLogSender(cfg *config.WAF) *remotelog.Sender { return remotelog.New(remotelog.Params{ URL: cfg.LogRemoteURL, RootCAs: cfg.LogRemoteTLSCAs, BufferLines: cfg.LogRemoteBufferLines, Facility: cfg.LogRemoteFacility, AppName: cfg.LogRemoteAppName, }) } // startSending runs remote until the function it returns is called, which // then waits at most remoteLogStopTimeout for the lines still waiting to // be sent. Sending goes on after ctx is done, so that the lines written // while sww stops are sent too. func startSending( ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger, ) func() { sending, stop := context.WithCancel(context.WithoutCancel(ctx)) sent := make(chan struct{}) go func() { remote.Run(sending, processLog) close(sent) }() return func() { stop() select { case <-sent: case <-time.After(remoteLogStopTimeout): } } }