sneak/routewatch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add queue high water marks to handler statistics

Browse Source 
- Track the maximum queue length seen for each handler
- Display high water marks on the status page with percentage
- Helps identify which handlers are experiencing queue pressure
This commit is contained in:
Jeffrey Paul 2025-07-29 02:46:53 +02:00
parent 2cfca78464
commit 23127b86e9
4 changed files with 109400 additions and 267764 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
internal/server/handlers.go
View File

@ -174,14 +174,15 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
func (s *Server) handleStats() http.HandlerFunc { func (s *Server) handleStats() http.HandlerFunc {
// HandlerStatsInfo represents handler statistics in the API response // HandlerStatsInfo represents handler statistics in the API response
type HandlerStatsInfo struct { type HandlerStatsInfo struct {
Name string `json:"name"` Name string `json:"name"`
QueueLength int `json:"queue_length"` QueueLength int `json:"queue_length"`
QueueCapacity int `json:"queue_capacity"` QueueCapacity int `json:"queue_capacity"`
ProcessedCount uint64 `json:"processed_count"` QueueHighWaterMark int `json:"queue_high_water_mark"`
DroppedCount uint64 `json:"dropped_count"` ProcessedCount uint64 `json:"processed_count"`
AvgProcessTimeMs float64 `json:"avg_process_time_ms"` DroppedCount uint64 `json:"dropped_count"`
MinProcessTimeMs float64 `json:"min_process_time_ms"` AvgProcessTimeMs float64 `json:"avg_process_time_ms"`
MaxProcessTimeMs float64 `json:"max_process_time_ms"` MinProcessTimeMs float64 `json:"min_process_time_ms"`
MaxProcessTimeMs float64 `json:"max_process_time_ms"`
} }
// StatsResponse represents the API statistics response // StatsResponse represents the API statistics response
@ -281,14 +282,15 @@ func (s *Server) handleStats() http.HandlerFunc {
const microsecondsPerMillisecond = 1000.0 const microsecondsPerMillisecond = 1000.0
for _, hs := range handlerStats { for _, hs := range handlerStats {
handlerStatsInfo = append(handlerStatsInfo, HandlerStatsInfo{ handlerStatsInfo = append(handlerStatsInfo, HandlerStatsInfo{
Name: hs.Name, Name: hs.Name,
QueueLength: hs.QueueLength, QueueLength: hs.QueueLength,
QueueCapacity: hs.QueueCapacity, QueueCapacity: hs.QueueCapacity,
ProcessedCount: hs.ProcessedCount, QueueHighWaterMark: hs.QueueHighWaterMark,
DroppedCount: hs.DroppedCount, ProcessedCount: hs.ProcessedCount,
AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / microsecondsPerMillisecond, DroppedCount: hs.DroppedCount,
MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / microsecondsPerMillisecond, AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / microsecondsPerMillisecond,
MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / microsecondsPerMillisecond, MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / microsecondsPerMillisecond,
MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / microsecondsPerMillisecond,
}) })
} }

52
internal/streamer/streamer.go
View File

@ -54,12 +54,13 @@ type RawMessageHandler func(line string)
// handlerMetrics tracks performance metrics for a handler // handlerMetrics tracks performance metrics for a handler
type handlerMetrics struct { type handlerMetrics struct {
processedCount uint64 // Total messages processed processedCount uint64 // Total messages processed
droppedCount uint64 // Total messages dropped droppedCount uint64 // Total messages dropped
totalTime time.Duration // Total processing time (for average calculation) totalTime time.Duration // Total processing time (for average calculation)
minTime time.Duration // Minimum processing time minTime time.Duration // Minimum processing time
maxTime time.Duration // Maximum processing time maxTime time.Duration // Maximum processing time
mu sync.Mutex // Protects the metrics queueHighWaterMark int // Maximum queue length seen
mu sync.Mutex // Protects the metrics
} }
// handlerInfo wraps a handler with its queue and metrics // handlerInfo wraps a handler with its queue and metrics
@ -214,14 +215,15 @@ func (s *Streamer) GetMetricsTracker() *metrics.Tracker {
// HandlerStats represents metrics for a single handler // HandlerStats represents metrics for a single handler
type HandlerStats struct { type HandlerStats struct {
Name string Name string
QueueLength int QueueLength int
QueueCapacity int QueueCapacity int
ProcessedCount uint64 QueueHighWaterMark int
DroppedCount uint64 ProcessedCount uint64
AvgProcessTime time.Duration DroppedCount uint64
MinProcessTime time.Duration AvgProcessTime time.Duration
MaxProcessTime time.Duration MinProcessTime time.Duration
MaxProcessTime time.Duration
} }
// GetHandlerStats returns current handler statistics // GetHandlerStats returns current handler statistics
@ -235,13 +237,14 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
info.metrics.mu.Lock() info.metrics.mu.Lock()
hs := HandlerStats{ hs := HandlerStats{
Name: fmt.Sprintf("%T", info.handler), Name: fmt.Sprintf("%T", info.handler),
QueueLength: len(info.queue), QueueLength: len(info.queue),
QueueCapacity: cap(info.queue), QueueCapacity: cap(info.queue),
ProcessedCount: info.metrics.processedCount, QueueHighWaterMark: info.metrics.queueHighWaterMark,
DroppedCount: info.metrics.droppedCount, ProcessedCount: info.metrics.processedCount,
MinProcessTime: info.metrics.minTime, DroppedCount: info.metrics.droppedCount,
MaxProcessTime: info.metrics.maxTime, MinProcessTime: info.metrics.minTime,
MaxProcessTime: info.metrics.maxTime,
} }
// Calculate average time // Calculate average time
@ -550,6 +553,13 @@ func (s *Streamer) stream(ctx context.Context) error {
select { select {
case info.queue <- &msg: case info.queue <- &msg:
// Message queued successfully // Message queued successfully
// Update high water mark if needed
queueLen := len(info.queue)
info.metrics.mu.Lock()
if queueLen > info.metrics.queueHighWaterMark {
info.metrics.queueHighWaterMark = queueLen
}
info.metrics.mu.Unlock()
default: default:
// Queue is full, drop the message // Queue is full, drop the message
atomic.AddUint64(&info.metrics.droppedCount, 1) atomic.AddUint64(&info.metrics.droppedCount, 1)

4
internal/templates/status.html
View File

@ -264,6 +264,10 @@
<span class="metric-label">Queue</span> <span class="metric-label">Queue</span>
<span class="metric-value">${handler.queue_length}/${handler.queue_capacity}</span> <span class="metric-value">${handler.queue_length}/${handler.queue_capacity}</span>
</div> </div>
<div class="metric">
<span class="metric-label">High Water Mark</span>
<span class="metric-value">${handler.queue_high_water_mark}/${handler.queue_capacity} (${Math.round(handler.queue_high_water_mark * 100 / handler.queue_capacity)}%)</span>
</div>
<div class="metric"> <div class="metric">
<span class="metric-label">Processed</span> <span class="metric-label">Processed</span>
<span class="metric-value">${formatNumber(handler.processed_count)}</span> <span class="metric-value">${formatNumber(handler.processed_count)}</span>

377074
log.txt
View File

File diff suppressed because it is too large Load Diff