Add WHOIS stats to status page with adaptive fetcher improvements
- Add WHOIS Fetcher card showing fresh/stale/never-fetched ASN counts - Display hourly success/error counts and current fetch interval - Increase max WHOIS rate to 1/sec (down from 10 sec minimum) - Select random stale ASN instead of oldest for better distribution - Add index on whois_updated_at for query performance - Track success/error timestamps for hourly stats - Add GetWHOISStats database method for freshness statistics
This commit is contained in:
@@ -58,6 +58,19 @@ func writeJSONSuccess(w http.ResponseWriter, data interface{}) error {
|
||||
})
|
||||
}
|
||||
|
||||
// WHOISStatsInfo contains WHOIS fetcher statistics for the status page.
|
||||
type WHOISStatsInfo struct {
|
||||
TotalASNs int `json:"total_asns"`
|
||||
FreshASNs int `json:"fresh_asns"`
|
||||
StaleASNs int `json:"stale_asns"`
|
||||
NeverFetched int `json:"never_fetched"`
|
||||
SuccessesLastHour int `json:"successes_last_hour"`
|
||||
ErrorsLastHour int `json:"errors_last_hour"`
|
||||
CurrentInterval string `json:"current_interval"`
|
||||
ConsecutiveFails int `json:"consecutive_fails"`
|
||||
FreshPercent float64 `json:"fresh_percent"`
|
||||
}
|
||||
|
||||
// handleStatusJSON returns a handler that serves JSON statistics including
|
||||
// uptime, message counts, database stats, and route information.
|
||||
func (s *Server) handleStatusJSON() http.HandlerFunc {
|
||||
@@ -88,6 +101,7 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
||||
IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
|
||||
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
|
||||
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
|
||||
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
|
||||
}
|
||||
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -149,6 +163,12 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
||||
var memStats runtime.MemStats
|
||||
runtime.ReadMemStats(&memStats)
|
||||
|
||||
// Get WHOIS stats if fetcher is available
|
||||
var whoisStats *WHOISStatsInfo
|
||||
if s.asnFetcher != nil {
|
||||
whoisStats = s.getWHOISStats(ctx)
|
||||
}
|
||||
|
||||
stats := Stats{
|
||||
Uptime: uptime,
|
||||
TotalMessages: metrics.TotalMessages,
|
||||
@@ -175,6 +195,7 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
||||
IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec,
|
||||
IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
|
||||
IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
|
||||
WHOISStats: whoisStats,
|
||||
}
|
||||
|
||||
if err := writeJSONSuccess(w, stats); err != nil {
|
||||
@@ -183,6 +204,44 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// getWHOISStats builds WHOIS statistics from database and fetcher.
|
||||
func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo {
|
||||
// Get database WHOIS stats
|
||||
dbStats, err := s.db.GetWHOISStats(ctx, whoisStaleThreshold)
|
||||
if err != nil {
|
||||
s.logger.Warn("Failed to get WHOIS stats", "error", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get fetcher stats
|
||||
fetcherStats := s.asnFetcher.GetStats()
|
||||
|
||||
// Calculate fresh percentage
|
||||
var freshPercent float64
|
||||
if dbStats.TotalASNs > 0 {
|
||||
freshPercent = float64(dbStats.FreshASNs) / float64(dbStats.TotalASNs) * percentMultiplier
|
||||
}
|
||||
|
||||
return &WHOISStatsInfo{
|
||||
TotalASNs: dbStats.TotalASNs,
|
||||
FreshASNs: dbStats.FreshASNs,
|
||||
StaleASNs: dbStats.StaleASNs,
|
||||
NeverFetched: dbStats.NeverFetched,
|
||||
SuccessesLastHour: fetcherStats.SuccessesLastHour,
|
||||
ErrorsLastHour: fetcherStats.ErrorsLastHour,
|
||||
CurrentInterval: fetcherStats.CurrentInterval.String(),
|
||||
ConsecutiveFails: fetcherStats.ConsecutiveFails,
|
||||
FreshPercent: freshPercent,
|
||||
}
|
||||
}
|
||||
|
||||
// whoisStaleThreshold matches the fetcher's threshold for consistency.
|
||||
const whoisStaleThreshold = 30 * 24 * time.Hour
|
||||
|
||||
// percentMultiplier converts a ratio to a percentage.
|
||||
const percentMultiplier = 100
|
||||
|
||||
// handleStats returns a handler that serves API v1 statistics including
|
||||
// detailed handler queue statistics and performance metrics.
|
||||
func (s *Server) handleStats() http.HandlerFunc {
|
||||
@@ -227,6 +286,7 @@ func (s *Server) handleStats() http.HandlerFunc {
|
||||
HandlerStats []HandlerStatsInfo `json:"handler_stats"`
|
||||
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
|
||||
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
|
||||
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
|
||||
}
|
||||
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -314,6 +374,12 @@ func (s *Server) handleStats() http.HandlerFunc {
|
||||
var memStats runtime.MemStats
|
||||
runtime.ReadMemStats(&memStats)
|
||||
|
||||
// Get WHOIS stats if fetcher is available
|
||||
var whoisStats *WHOISStatsInfo
|
||||
if s.asnFetcher != nil {
|
||||
whoisStats = s.getWHOISStats(ctx)
|
||||
}
|
||||
|
||||
stats := StatsResponse{
|
||||
Uptime: uptime,
|
||||
TotalMessages: metrics.TotalMessages,
|
||||
@@ -341,6 +407,7 @@ func (s *Server) handleStats() http.HandlerFunc {
|
||||
HandlerStats: handlerStatsInfo,
|
||||
IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
|
||||
IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
|
||||
WHOISStats: whoisStats,
|
||||
}
|
||||
|
||||
if err := writeJSONSuccess(w, stats); err != nil {
|
||||
|
||||
@@ -13,9 +13,18 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
// ASNFetcherStats contains WHOIS fetcher statistics.
|
||||
type ASNFetcherStats struct {
|
||||
SuccessesLastHour int
|
||||
ErrorsLastHour int
|
||||
CurrentInterval time.Duration
|
||||
ConsecutiveFails int
|
||||
}
|
||||
|
||||
// ASNFetcher is an interface for queuing ASN WHOIS lookups.
|
||||
type ASNFetcher interface {
|
||||
QueueImmediate(asn int)
|
||||
GetStats() ASNFetcherStats
|
||||
}
|
||||
|
||||
// Server provides HTTP endpoints for status monitoring
|
||||
|
||||
Reference in New Issue
Block a user