Add WHOIS stats to status page with adaptive fetcher improvements

- Add WHOIS Fetcher card showing fresh/stale/never-fetched ASN counts
- Display hourly success/error counts and current fetch interval
- Increase max WHOIS rate to 1/sec (down from 10 sec minimum)
- Select random stale ASN instead of oldest for better distribution
- Add index on whois_updated_at for query performance
- Track success/error timestamps for hourly stats
- Add GetWHOISStats database method for freshness statistics
This commit is contained in:
2025-12-27 16:20:09 +07:00
parent f8b7d3b773
commit d2041a5a55
8 changed files with 252 additions and 15 deletions

View File

@@ -58,6 +58,19 @@ func writeJSONSuccess(w http.ResponseWriter, data interface{}) error {
})
}
// WHOISStatsInfo contains WHOIS fetcher statistics for the status page.
type WHOISStatsInfo struct {
TotalASNs int `json:"total_asns"`
FreshASNs int `json:"fresh_asns"`
StaleASNs int `json:"stale_asns"`
NeverFetched int `json:"never_fetched"`
SuccessesLastHour int `json:"successes_last_hour"`
ErrorsLastHour int `json:"errors_last_hour"`
CurrentInterval string `json:"current_interval"`
ConsecutiveFails int `json:"consecutive_fails"`
FreshPercent float64 `json:"fresh_percent"`
}
// handleStatusJSON returns a handler that serves JSON statistics including
// uptime, message counts, database stats, and route information.
func (s *Server) handleStatusJSON() http.HandlerFunc {
@@ -88,6 +101,7 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
}
return func(w http.ResponseWriter, r *http.Request) {
@@ -149,6 +163,12 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
// Get WHOIS stats if fetcher is available
var whoisStats *WHOISStatsInfo
if s.asnFetcher != nil {
whoisStats = s.getWHOISStats(ctx)
}
stats := Stats{
Uptime: uptime,
TotalMessages: metrics.TotalMessages,
@@ -175,6 +195,7 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec,
IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
WHOISStats: whoisStats,
}
if err := writeJSONSuccess(w, stats); err != nil {
@@ -183,6 +204,44 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
}
}
// getWHOISStats builds WHOIS statistics from database and fetcher.
func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo {
// Get database WHOIS stats
dbStats, err := s.db.GetWHOISStats(ctx, whoisStaleThreshold)
if err != nil {
s.logger.Warn("Failed to get WHOIS stats", "error", err)
return nil
}
// Get fetcher stats
fetcherStats := s.asnFetcher.GetStats()
// Calculate fresh percentage
var freshPercent float64
if dbStats.TotalASNs > 0 {
freshPercent = float64(dbStats.FreshASNs) / float64(dbStats.TotalASNs) * percentMultiplier
}
return &WHOISStatsInfo{
TotalASNs: dbStats.TotalASNs,
FreshASNs: dbStats.FreshASNs,
StaleASNs: dbStats.StaleASNs,
NeverFetched: dbStats.NeverFetched,
SuccessesLastHour: fetcherStats.SuccessesLastHour,
ErrorsLastHour: fetcherStats.ErrorsLastHour,
CurrentInterval: fetcherStats.CurrentInterval.String(),
ConsecutiveFails: fetcherStats.ConsecutiveFails,
FreshPercent: freshPercent,
}
}
// whoisStaleThreshold matches the fetcher's threshold for consistency.
const whoisStaleThreshold = 30 * 24 * time.Hour
// percentMultiplier converts a ratio to a percentage.
const percentMultiplier = 100
// handleStats returns a handler that serves API v1 statistics including
// detailed handler queue statistics and performance metrics.
func (s *Server) handleStats() http.HandlerFunc {
@@ -227,6 +286,7 @@ func (s *Server) handleStats() http.HandlerFunc {
HandlerStats []HandlerStatsInfo `json:"handler_stats"`
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
}
return func(w http.ResponseWriter, r *http.Request) {
@@ -314,6 +374,12 @@ func (s *Server) handleStats() http.HandlerFunc {
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
// Get WHOIS stats if fetcher is available
var whoisStats *WHOISStatsInfo
if s.asnFetcher != nil {
whoisStats = s.getWHOISStats(ctx)
}
stats := StatsResponse{
Uptime: uptime,
TotalMessages: metrics.TotalMessages,
@@ -341,6 +407,7 @@ func (s *Server) handleStats() http.HandlerFunc {
HandlerStats: handlerStatsInfo,
IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
WHOISStats: whoisStats,
}
if err := writeJSONSuccess(w, stats); err != nil {

View File

@@ -13,9 +13,18 @@ import (
"github.com/go-chi/chi/v5"
)
// ASNFetcherStats contains WHOIS fetcher statistics.
type ASNFetcherStats struct {
SuccessesLastHour int
ErrorsLastHour int
CurrentInterval time.Duration
ConsecutiveFails int
}
// ASNFetcher is an interface for queuing ASN WHOIS lookups.
type ASNFetcher interface {
QueueImmediate(asn int)
GetStats() ASNFetcherStats
}
// Server provides HTTP endpoints for status monitoring