Add queue high water marks to handler statistics

- Track the maximum queue length seen for each handler
- Display high water marks on the status page with percentage
- Helps identify which handlers are experiencing queue pressure

internal/streamer/streamer.go

@@ -54,12 +54,13 @@ type RawMessageHandler func(line string)
 
 // handlerMetrics tracks performance metrics for a handler
 type handlerMetrics struct {
-	processedCount uint64        // Total messages processed
-	droppedCount   uint64        // Total messages dropped
-	totalTime      time.Duration // Total processing time (for average calculation)
-	minTime        time.Duration // Minimum processing time
-	maxTime        time.Duration // Maximum processing time
-	mu             sync.Mutex    // Protects the metrics
+	processedCount     uint64        // Total messages processed
+	droppedCount       uint64        // Total messages dropped
+	totalTime          time.Duration // Total processing time (for average calculation)
+	minTime            time.Duration // Minimum processing time
+	maxTime            time.Duration // Maximum processing time
+	queueHighWaterMark int           // Maximum queue length seen
+	mu                 sync.Mutex    // Protects the metrics
 }
 
 // handlerInfo wraps a handler with its queue and metrics
@@ -214,14 +215,15 @@ func (s *Streamer) GetMetricsTracker() *metrics.Tracker {
 
 // HandlerStats represents metrics for a single handler
 type HandlerStats struct {
-	Name           string
-	QueueLength    int
-	QueueCapacity  int
-	ProcessedCount uint64
-	DroppedCount   uint64
-	AvgProcessTime time.Duration
-	MinProcessTime time.Duration
-	MaxProcessTime time.Duration
+	Name               string
+	QueueLength        int
+	QueueCapacity      int
+	QueueHighWaterMark int
+	ProcessedCount     uint64
+	DroppedCount       uint64
+	AvgProcessTime     time.Duration
+	MinProcessTime     time.Duration
+	MaxProcessTime     time.Duration
 }
 
 // GetHandlerStats returns current handler statistics
@@ -235,13 +237,14 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
 		info.metrics.mu.Lock()
 
 		hs := HandlerStats{
-			Name:           fmt.Sprintf("%T", info.handler),
-			QueueLength:    len(info.queue),
-			QueueCapacity:  cap(info.queue),
-			ProcessedCount: info.metrics.processedCount,
-			DroppedCount:   info.metrics.droppedCount,
-			MinProcessTime: info.metrics.minTime,
-			MaxProcessTime: info.metrics.maxTime,
+			Name:               fmt.Sprintf("%T", info.handler),
+			QueueLength:        len(info.queue),
+			QueueCapacity:      cap(info.queue),
+			QueueHighWaterMark: info.metrics.queueHighWaterMark,
+			ProcessedCount:     info.metrics.processedCount,
+			DroppedCount:       info.metrics.droppedCount,
+			MinProcessTime:     info.metrics.minTime,
+			MaxProcessTime:     info.metrics.maxTime,
 		}
 
 		// Calculate average time
@@ -550,6 +553,13 @@ func (s *Streamer) stream(ctx context.Context) error {
 			select {
 			case info.queue <- &msg:
 				// Message queued successfully
+				// Update high water mark if needed
+				queueLen := len(info.queue)
+				info.metrics.mu.Lock()
+				if queueLen > info.metrics.queueHighWaterMark {
+					info.metrics.queueHighWaterMark = queueLen
+				}
+				info.metrics.mu.Unlock()
 			default:
 				// Queue is full, drop the message
 				atomic.AddUint64(&info.metrics.droppedCount, 1)