From 280c8e1fc7a11ac6afa32262f529ff8a8da83f51 Mon Sep 17 00:00:00 2001 From: sneak Date: Mon, 21 Sep 2026 14:04:41 +0000 Subject: [PATCH] Shrink the four handler queues from 100,000 to 20,000 (closes #11) Each handler queue held 100,000 message pointers; a message is retained until the slowest handler drains it, so all four full was a derived worst case near 800 MiB. Twenty thousand is about four seconds of feed at peak and caps that at roughly 160 MiB. The streamer already drops rather than blocks on a full queue, so the smaller bound is safe. Batch sizes are unchanged. The largest, asnBatchSize, is 30,000 and now exceeds its queue, but each queued message contributes every ASN in its path, and every handler also flushes on its own timer regardless of fill, so batches still flush and no size change is warranted. Model: opus-4-8 --- internal/routewatch/ashandler.go | 8 +++++--- internal/routewatch/peerhandler.go | 7 +++++-- internal/routewatch/peeringhandler.go | 8 ++++---- internal/routewatch/prefixhandler.go | 8 +++++--- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/internal/routewatch/ashandler.go b/internal/routewatch/ashandler.go index c1ff34f..3eae6f3 100644 --- a/internal/routewatch/ashandler.go +++ b/internal/routewatch/ashandler.go @@ -10,9 +10,11 @@ import ( ) const ( - // asHandlerQueueSize is the queue capacity for ASN operations - // DO NOT set this higher than 100000 without explicit instructions - asHandlerQueueSize = 100000 + // asHandlerQueueSize is the queue capacity for ASN operations, about 4 + // seconds of feed at peak. The streamer drops rather than blocks when a + // queue is full, so this bounds memory. Batches still flush on a timer + // (asnBatchTimeout), so a queue smaller than asnBatchSize is fine. + asHandlerQueueSize = 20000 // asnBatchSize is the number of ASN operations to batch together asnBatchSize = 30000 diff --git a/internal/routewatch/peerhandler.go b/internal/routewatch/peerhandler.go index 7ffabab..2f7bf70 100644 --- a/internal/routewatch/peerhandler.go +++ b/internal/routewatch/peerhandler.go @@ -14,8 +14,11 @@ import ( ) const ( - // peerHandlerQueueSize is the queue capacity for peer tracking operations - peerHandlerQueueSize = 100000 + // peerHandlerQueueSize is the queue capacity for peer tracking operations, + // about 4 seconds of feed at peak. The streamer drops rather than blocks + // when a queue is full, so this bounds memory. Batches still flush on a + // timer (peerBatchTimeout). + peerHandlerQueueSize = 20000 // peerBatchSize is the number of peer updates to batch together peerBatchSize = 10000 diff --git a/internal/routewatch/peeringhandler.go b/internal/routewatch/peeringhandler.go index 820ccde..b4f3277 100644 --- a/internal/routewatch/peeringhandler.go +++ b/internal/routewatch/peeringhandler.go @@ -11,10 +11,10 @@ import ( ) const ( - // peeringHandlerQueueSize defines the buffer capacity for the peering - // handler's message queue. This should be large enough to handle bursts - // of BGP UPDATE messages without blocking. - peeringHandlerQueueSize = 100000 + // peeringHandlerQueueSize is the buffer capacity for the peering handler's + // message queue, about 4 seconds of feed at peak. The streamer drops + // rather than blocks when a queue is full, so this bounds memory. + peeringHandlerQueueSize = 20000 // minPathLengthForPeering specifies the minimum number of ASNs required // in a BGP AS path to extract peering relationships. A path with fewer diff --git a/internal/routewatch/prefixhandler.go b/internal/routewatch/prefixhandler.go index 4223b20..9ce5675 100644 --- a/internal/routewatch/prefixhandler.go +++ b/internal/routewatch/prefixhandler.go @@ -14,9 +14,11 @@ import ( ) const ( - // prefixHandlerQueueSize is the queue capacity for prefix tracking operations - // DO NOT set this higher than 100000 without explicit instructions - prefixHandlerQueueSize = 100000 + // prefixHandlerQueueSize is the queue capacity for prefix tracking + // operations, about 4 seconds of feed at peak. The streamer drops rather + // than blocks when a queue is full, so this bounds memory. Batches still + // flush on a timer (prefixBatchTimeout). + prefixHandlerQueueSize = 20000 // prefixBatchSize is the number of prefix updates to batch together prefixBatchSize = 25000 -- 2.54.0