Drop an update only when its handler's queue is full (closes #229)

The streamer dropped a random share of a handler's updates once its
queue was half full. When the feed paused and then sent its backlog
while the routing table's Overview walked every prefix, the prefix
handler lost about 10,000 updates with its queue never full.

The streamer now drops an update only when the queue is full. The
prefix handler's queue holds 100,000 messages, about 100 MiB, so it
can hold such a backlog. The README's Memory and Feed handling
sections give the new sizes and the rule.

Model: opus-5-5

internal/streamer/streamer_internal_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
@@ -258,6 +259,65 @@ func TestHeldStateMessagesKeepTheirPlaceInTheQueue(t *testing.T) {
 	}
 }
 
+// burstQueueCapacity is the size of a burstHandler's queue.
+const burstQueueCapacity = 1000
+
+// burstHandler wants UPDATE messages and has a queue of burstQueueCapacity.
+// Tests read what it is handed from its queue, so HandleMessage does nothing.
+type burstHandler struct{}
+
+func (burstHandler) WantsMessage(messageType string) bool {
+	return messageType == ristypes.MessageTypeUpdate
+}
+
+func (burstHandler) QueueCapacity() int { return burstQueueCapacity }
+
+func (burstHandler) HandleMessage(*ristypes.RISMessage) {}
+
+// TestUpdatesDroppedOnlyWhenTheQueueIsFull has the server send one UPDATE more
+// than the handler's queue holds, each with its own timestamp. Nothing takes
+// messages off the queue meanwhile. The streamer must queue the updates in
+// feed order until the queue is full, and drop only the last one.
+func TestUpdatesDroppedOnlyWhenTheQueueIsFull(t *testing.T) {
+	t.Parallel()
+
+	const (
+		firstTimestamp = 1700000000
+		lineFormat     = `{"type":"ris_message","data":{"type":"UPDATE",` +
+			`"peer":"192.0.2.1","peer_asn":"64496","timestamp":%d}}` + "\n"
+	)
+
+	var lines strings.Builder
+
+	for i := range burstQueueCapacity + 1 {
+		fmt.Fprintf(&lines, lineFormat, firstTimestamp+i)
+	}
+
+	s := New(logger.New(), metrics.New())
+	s.RegisterHandler(burstHandler{})
+
+	streamLines(t, s, lines.String())
+
+	dropped := s.GetHandlerStats()[0].DroppedCount
+	if dropped != 1 {
+		t.Errorf("%d updates dropped, want only the one that found the queue "+
+			"full", dropped)
+	}
+
+	queue := s.handlers[0].queue
+	if len(queue) != burstQueueCapacity {
+		t.Fatalf("%d updates queued, want %d", len(queue), burstQueueCapacity)
+	}
+
+	for i := range burstQueueCapacity {
+		msg := <-queue
+		if msg.Timestamp != float64(firstTimestamp+i) {
+			t.Fatalf("update %d on the queue has timestamp %.0f, want %d",
+				i, msg.Timestamp, firstTimestamp+i)
+		}
+	}
+}
+
 // takeAll takes the messages off queue, as a handler's worker would, and sends
 // them on the returned channel once queue is closed.
 func takeAll(queue chan *ristypes.RISMessage) chan []*ristypes.RISMessage {