Add queue high water marks to handler statistics
- Track the maximum queue length seen for each handler - Display high water marks on the status page with percentage - Helps identify which handlers are experiencing queue pressure
This commit is contained in:
parent
2cfca78464
commit
23127b86e9
@ -174,14 +174,15 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
||||
func (s *Server) handleStats() http.HandlerFunc {
|
||||
// HandlerStatsInfo represents handler statistics in the API response
|
||||
type HandlerStatsInfo struct {
|
||||
Name string `json:"name"`
|
||||
QueueLength int `json:"queue_length"`
|
||||
QueueCapacity int `json:"queue_capacity"`
|
||||
ProcessedCount uint64 `json:"processed_count"`
|
||||
DroppedCount uint64 `json:"dropped_count"`
|
||||
AvgProcessTimeMs float64 `json:"avg_process_time_ms"`
|
||||
MinProcessTimeMs float64 `json:"min_process_time_ms"`
|
||||
MaxProcessTimeMs float64 `json:"max_process_time_ms"`
|
||||
Name string `json:"name"`
|
||||
QueueLength int `json:"queue_length"`
|
||||
QueueCapacity int `json:"queue_capacity"`
|
||||
QueueHighWaterMark int `json:"queue_high_water_mark"`
|
||||
ProcessedCount uint64 `json:"processed_count"`
|
||||
DroppedCount uint64 `json:"dropped_count"`
|
||||
AvgProcessTimeMs float64 `json:"avg_process_time_ms"`
|
||||
MinProcessTimeMs float64 `json:"min_process_time_ms"`
|
||||
MaxProcessTimeMs float64 `json:"max_process_time_ms"`
|
||||
}
|
||||
|
||||
// StatsResponse represents the API statistics response
|
||||
@ -281,14 +282,15 @@ func (s *Server) handleStats() http.HandlerFunc {
|
||||
const microsecondsPerMillisecond = 1000.0
|
||||
for _, hs := range handlerStats {
|
||||
handlerStatsInfo = append(handlerStatsInfo, HandlerStatsInfo{
|
||||
Name: hs.Name,
|
||||
QueueLength: hs.QueueLength,
|
||||
QueueCapacity: hs.QueueCapacity,
|
||||
ProcessedCount: hs.ProcessedCount,
|
||||
DroppedCount: hs.DroppedCount,
|
||||
AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / microsecondsPerMillisecond,
|
||||
MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / microsecondsPerMillisecond,
|
||||
MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / microsecondsPerMillisecond,
|
||||
Name: hs.Name,
|
||||
QueueLength: hs.QueueLength,
|
||||
QueueCapacity: hs.QueueCapacity,
|
||||
QueueHighWaterMark: hs.QueueHighWaterMark,
|
||||
ProcessedCount: hs.ProcessedCount,
|
||||
DroppedCount: hs.DroppedCount,
|
||||
AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / microsecondsPerMillisecond,
|
||||
MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / microsecondsPerMillisecond,
|
||||
MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / microsecondsPerMillisecond,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -54,12 +54,13 @@ type RawMessageHandler func(line string)
|
||||
|
||||
// handlerMetrics tracks performance metrics for a handler
|
||||
type handlerMetrics struct {
|
||||
processedCount uint64 // Total messages processed
|
||||
droppedCount uint64 // Total messages dropped
|
||||
totalTime time.Duration // Total processing time (for average calculation)
|
||||
minTime time.Duration // Minimum processing time
|
||||
maxTime time.Duration // Maximum processing time
|
||||
mu sync.Mutex // Protects the metrics
|
||||
processedCount uint64 // Total messages processed
|
||||
droppedCount uint64 // Total messages dropped
|
||||
totalTime time.Duration // Total processing time (for average calculation)
|
||||
minTime time.Duration // Minimum processing time
|
||||
maxTime time.Duration // Maximum processing time
|
||||
queueHighWaterMark int // Maximum queue length seen
|
||||
mu sync.Mutex // Protects the metrics
|
||||
}
|
||||
|
||||
// handlerInfo wraps a handler with its queue and metrics
|
||||
@ -214,14 +215,15 @@ func (s *Streamer) GetMetricsTracker() *metrics.Tracker {
|
||||
|
||||
// HandlerStats represents metrics for a single handler
|
||||
type HandlerStats struct {
|
||||
Name string
|
||||
QueueLength int
|
||||
QueueCapacity int
|
||||
ProcessedCount uint64
|
||||
DroppedCount uint64
|
||||
AvgProcessTime time.Duration
|
||||
MinProcessTime time.Duration
|
||||
MaxProcessTime time.Duration
|
||||
Name string
|
||||
QueueLength int
|
||||
QueueCapacity int
|
||||
QueueHighWaterMark int
|
||||
ProcessedCount uint64
|
||||
DroppedCount uint64
|
||||
AvgProcessTime time.Duration
|
||||
MinProcessTime time.Duration
|
||||
MaxProcessTime time.Duration
|
||||
}
|
||||
|
||||
// GetHandlerStats returns current handler statistics
|
||||
@ -235,13 +237,14 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
|
||||
info.metrics.mu.Lock()
|
||||
|
||||
hs := HandlerStats{
|
||||
Name: fmt.Sprintf("%T", info.handler),
|
||||
QueueLength: len(info.queue),
|
||||
QueueCapacity: cap(info.queue),
|
||||
ProcessedCount: info.metrics.processedCount,
|
||||
DroppedCount: info.metrics.droppedCount,
|
||||
MinProcessTime: info.metrics.minTime,
|
||||
MaxProcessTime: info.metrics.maxTime,
|
||||
Name: fmt.Sprintf("%T", info.handler),
|
||||
QueueLength: len(info.queue),
|
||||
QueueCapacity: cap(info.queue),
|
||||
QueueHighWaterMark: info.metrics.queueHighWaterMark,
|
||||
ProcessedCount: info.metrics.processedCount,
|
||||
DroppedCount: info.metrics.droppedCount,
|
||||
MinProcessTime: info.metrics.minTime,
|
||||
MaxProcessTime: info.metrics.maxTime,
|
||||
}
|
||||
|
||||
// Calculate average time
|
||||
@ -550,6 +553,13 @@ func (s *Streamer) stream(ctx context.Context) error {
|
||||
select {
|
||||
case info.queue <- &msg:
|
||||
// Message queued successfully
|
||||
// Update high water mark if needed
|
||||
queueLen := len(info.queue)
|
||||
info.metrics.mu.Lock()
|
||||
if queueLen > info.metrics.queueHighWaterMark {
|
||||
info.metrics.queueHighWaterMark = queueLen
|
||||
}
|
||||
info.metrics.mu.Unlock()
|
||||
default:
|
||||
// Queue is full, drop the message
|
||||
atomic.AddUint64(&info.metrics.droppedCount, 1)
|
||||
|
||||
@ -264,6 +264,10 @@
|
||||
<span class="metric-label">Queue</span>
|
||||
<span class="metric-value">${handler.queue_length}/${handler.queue_capacity}</span>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<span class="metric-label">High Water Mark</span>
|
||||
<span class="metric-value">${handler.queue_high_water_mark}/${handler.queue_capacity} (${Math.round(handler.queue_high_water_mark * 100 / handler.queue_capacity)}%)</span>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<span class="metric-label">Processed</span>
|
||||
<span class="metric-value">${formatNumber(handler.processed_count)}</span>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user