Keep taking messages while a peer's session goes down (closes #216)

The table's PeerDown visits every prefix under the write lock, which
takes over a second on a full table. The prefix handler called it
inline, so its queue filled and the streamer dropped updates.

PeerDown now holds the lock for 10,000 prefixes at a time, then lets
the lock go for as long again. The handler runs PeerDown on a separate
goroutine, one peer at a time. The handler holds back the down peer's
later messages until that PeerDown returns. Then the handler applies
them in order, one at a time, and takes new messages in between. It
holds back at most 20,000 messages. Stop waits for all of this before
its last flush.

Model: opus-5-5

internal/routewatch/prefixhandler_internal_test.go

@@ -237,6 +237,83 @@ func TestPrefixHandlerStopLogsNothingWhenItWroteEverything(t *testing.T) {
 	}
 }
 
+// TestPrefixHandlerStopLogsTheMessagesHeldBack stops a prefix handler with a
+// deadline that has passed, while a peer's PeerDown is held up and one message
+// of the peer is held back. Stop must log that message as not applied.
+func TestPrefixHandlerStopLogsTheMessagesHeldBack(t *testing.T) {
+	t.Parallel()
+
+	var logs bytes.Buffer
+
+	h := newPrefixHandler(newMockStore(), newErrorLogger(&logs),
+		routetable.New(routetable.Params{}), longerThanAnyTest,
+		maxTableChangesPerWrite)
+
+	h.peerDownMu.Lock()
+	defer h.peerDownMu.Unlock()
+
+	h.HandleMessage(peerAnnouncement(testPeerIP, testV4Prefix))
+	h.HandleMessage(sessionDown())
+	h.HandleMessage(peerAnnouncement(testPeerIP, testV6Prefix))
+
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	h.Stop(ctx)
+
+	logged := lastLogged(t, &logs)
+	if logged.HeldMessages != 1 {
+		t.Errorf("Stop logged %d held messages not applied, want 1",
+			logged.HeldMessages)
+	}
+}
+
+// TestPrefixHandlerStopWaitsForTheMessagesHeldBack stops a prefix handler
+// while a peer's PeerDown is held up and an announcement of the peer is held
+// back. Stop must wait. Once the PeerDown is let go and Stop returns, the
+// announcement must be in the database.
+func TestPrefixHandlerStopWaitsForTheMessagesHeldBack(t *testing.T) {
+	t.Parallel()
+
+	db := newTestDatabase(t)
+	h := newPrefixHandler(db, logger.New(), routetable.New(routetable.Params{}),
+		longerThanAnyTest, maxTableChangesPerWrite)
+
+	h.peerDownMu.Lock()
+	h.HandleMessage(sessionDown())
+	h.HandleMessage(peerAnnouncement(testPeerIP, testV4Prefix))
+
+	stopped := make(chan struct{})
+
+	go func() {
+		h.Stop(t.Context())
+		close(stopped)
+	}()
+
+	select {
+	case <-stopped:
+		t.Error("Stop returned while a PeerDown was held up")
+	case <-time.After(stillWaitingAfter):
+	}
+
+	h.peerDownMu.Unlock()
+
+	select {
+	case <-stopped:
+	case <-time.After(stepTimeout):
+		t.Fatalf("Stop did not return within %s of the PeerDown's release",
+			stepTimeout)
+	}
+
+	read, err := db.ReadRouteTable(t.Context())
+	if err != nil {
+		t.Fatalf("ReadRouteTable: %v", err)
+	}
+
+	checkTableOrigins(t, read, map[string]map[int][]string{
+		testV4Prefix: {testASNB: {testPeerIP}},
+	})
+}
+
 // longerThanAnyTest is a timer interval that no test lasts. A test gives it to
 // newPrefixHandler, newASHandler, newPeerHandler or newPeeringHandler when the
 // handler must write only when its batch is full, when the test says, or on
@@ -253,6 +330,7 @@ func prefixHandlerWithoutTimer(store database.Store) *PrefixHandler {
 		table:                routetable.New(routetable.Params{}),
 		batchMaxSize:         prefixBatchMaxSize,
 		tableChangesPerWrite: maxTableChangesPerWrite,
+		heldBack:             make(map[string][]*ristypes.RISMessage),
 	}
 }