Keep a peer-down until its route delete succeeds (closes #101)

When the delete of a peer's live routes after its session went down
failed, the down was dropped and the routes stayed. The prefix handler
now keeps such a peer, once, with the updates it sent after the down,
for the next flush, which retries the delete before it writes them; each
failure is logged. At most the last 25,000 of those updates are kept,
outside the batch, so they do not start a flush. The delete runs in
chunks of at most 10,000 routes per table, taken in prefix order
through the peer's key, each chunk in its own transaction, so the
database lock is released between chunks and a failure undoes at most
one chunk.

Model: opus-5-5

internal/routewatch/prefixhandler.go

@@ -28,7 +28,9 @@ const (
 	// prefixBatchMaxSize is the most prefix updates the batch holds while the
 	// flush goroutine writes the previous one: ten batches, about 60 MiB
 	// (README "Memory"). At this size HandleMessage waits until the flush
-	// goroutine takes the batch.
+	// goroutine takes the batch. On top of these, the handler holds at most
+	// prefixBatchSize updates, about 6 MiB, that a flush kept for the peers
+	// whose route delete failed (held).
 	prefixBatchMaxSize = 250000
 
 	// prefixBatchTimeout is the maximum time to wait before flushing a batch
@@ -48,13 +50,21 @@ type PrefixHandler struct {
 	logger  *logger.Logger
 	metrics *metrics.Tracker
 
-	// Batching. mu guards batch and peersDown only: the flush goroutine takes
-	// both under mu and writes them after releasing mu, so HandleMessage
-	// waits for a database write only while batch is at prefixBatchMaxSize.
+	// Batching. mu guards batch, held and peersDown only: the flush goroutine
+	// takes them under mu and writes them after releasing mu, so
+	// HandleMessage waits for a database write only while batch is at
+	// prefixBatchMaxSize.
 	mu    sync.Mutex
 	batch []prefixUpdate
-	// peersDown holds the peers whose BGP session went down since the batch
-	// was taken.
+	// held holds the updates that the peers whose route delete failed in the
+	// last flush sent after their session went down, for the next flush to
+	// write after it retries the delete. They are not in batch, so they do
+	// not count toward a full batch: a delete that keeps failing is retried
+	// at the pace of the flush timer.
+	held []prefixUpdate
+	// peersDown holds, each once, the peers whose BGP session went down
+	// since the batch was taken, and those whose route delete failed in the
+	// last flush.
 	peersDown []string
 	// batchTaken is signalled when the flush goroutine takes batch; its L is
 	// &mu.
@@ -200,17 +210,21 @@ func (h *PrefixHandler) Stop() {
 
 // peerSessionDown notes that the peer's BGP session went down, so that the next
 // flush deletes all of the peer's live routes before it writes the batch. The
-// peer's updates still in the batch came before the down, which replaces them,
-// so they are dropped; updates the peer sends later stay in the batch and are
-// written after the delete.
+// peer's updates still in the batch or held came before the down, which
+// replaces them, so they are dropped; updates the peer sends later stay in the
+// batch and are written after the delete. A peer already in peersDown is not
+// added again: one delete removes all of its routes.
 func (h *PrefixHandler) peerSessionDown(peer string) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
-	h.batch = slices.DeleteFunc(h.batch, func(update prefixUpdate) bool {
-		return update.peer == peer
-	})
-	h.peersDown = append(h.peersDown, peer)
+	isPeers := func(update prefixUpdate) bool { return update.peer == peer }
+	h.batch = slices.DeleteFunc(h.batch, isPeers)
+	h.held = slices.DeleteFunc(h.held, isPeers)
+
+	if !slices.Contains(h.peersDown, peer) {
+		h.peersDown = append(h.peersDown, peer)
+	}
 }
 
 // flushLoop runs in a goroutine, the only one that writes the batch: it
@@ -238,18 +252,23 @@ func (h *PrefixHandler) flushLoop() {
 	}
 }
 
-// takeBatch returns the pending updates and the peers whose session went down
-// since the last flush, and leaves an empty batch and no peers in their place
-// for HandleMessage to fill.
+// takeBatch returns the pending updates, the held ones first, and the peers in
+// peersDown, and leaves an empty batch, no held updates and no peers in their
+// place for HandleMessage to fill.
 func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
-	if len(h.batch) == 0 && len(h.peersDown) == 0 {
+	if len(h.held) == 0 && len(h.batch) == 0 && len(h.peersDown) == 0 {
 		return nil, nil
 	}
 
 	batch, peersDown := h.batch, h.peersDown
+	if len(h.held) > 0 {
+		batch = slices.Concat(h.held, h.batch)
+	}
+
+	h.held = nil
 	h.batch = make([]prefixUpdate, 0, prefixBatchSize)
 	h.peersDown = nil
 	h.batchTaken.Broadcast()
@@ -269,8 +288,12 @@ func (h *PrefixHandler) flushBatch() {
 	batchSize := len(batch)
 
 	// The routes of a peer whose session went down are deleted first: the
-	// batch holds only the updates the peer sent after the down.
-	h.deleteLiveRoutesOfPeers(peersDown)
+	// batch holds only the updates the peer sent after the down. A peer whose
+	// delete failed is kept, with those updates, for the next flush to retry.
+	failedPeers := h.deleteLiveRoutesOfPeers(peersDown)
+	if len(failedPeers) > 0 {
+		batch = h.keepForNextFlush(batch, failedPeers)
+	}
 
 	// For each prefix and peer, keep only the last update the feed delivered.
 	// Updates from different peers for the same prefix are all kept. Arrival
@@ -344,15 +367,21 @@ func (h *PrefixHandler) flushBatch() {
 }
 
 // deleteLiveRoutesOfPeers deletes all live routes of each of peers, the peers
-// whose BGP session went down. Most downs come from peers that have no routes,
-// so only a delete that removed routes is logged.
-func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) {
+// whose BGP session went down, and returns those whose delete failed. Most
+// downs come from peers that have no routes, so only a delete that removed
+// routes is logged.
+func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) []string {
+	var failed []string
+
 	for _, peer := range peers {
 		deleted, err := h.db.DeleteLiveRoutesOfPeer(peer)
 		if err != nil {
 			h.logger.Error(
-				"Failed to delete the live routes of a peer whose session went down",
-				"peer", peer, "error", err)
+				"Failed to delete the live routes of a peer whose session "+
+					"went down; the next flush retries",
+				"peer", peer, "routes_deleted", deleted, "error", err)
+
+			failed = append(failed, peer)
 
 			continue
 		}
@@ -363,6 +392,63 @@ func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) {
 				"peer", peer, "routes", deleted)
 		}
 	}
+
+	return failed
+}
+
+// keepForNextFlush puts the peers whose route delete failed back in peersDown,
+// and their updates from batch in held, so that the next flush retries each
+// delete before it writes the peer's updates; written now, they would be
+// deleted by the retry. The updates of a peer whose session went down again
+// since the batch was taken came before that down, so they are dropped, as
+// peerSessionDown drops them, and the peer is already in peersDown. It returns
+// the updates of the other peers, for this flush to write.
+//
+// While a delete keeps failing, the held updates would grow with every flush,
+// so only the last prefixBatchSize of them are kept and the older ones are
+// dropped. This never leaves in place a route the peer withdrew: a prefix with
+// a kept update keeps its last one, and the retry deletes all of the peer's
+// routes, so a prefix whose updates were all dropped has no route from the peer
+// until the peer announces it again.
+func (h *PrefixHandler) keepForNextFlush(
+	batch []prefixUpdate, failedPeers []string,
+) []prefixUpdate {
+	var kept, toWrite []prefixUpdate
+
+	for _, update := range batch {
+		if slices.Contains(failedPeers, update.peer) {
+			kept = append(kept, update)
+		} else {
+			toWrite = append(toWrite, update)
+		}
+	}
+
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	kept = slices.DeleteFunc(kept, func(update prefixUpdate) bool {
+		return slices.Contains(h.peersDown, update.peer)
+	})
+
+	if len(kept) > prefixBatchSize {
+		h.logger.Error(
+			"Dropped the oldest updates of peers whose route delete failed",
+			"peers", failedPeers, "dropped", len(kept)-prefixBatchSize)
+
+		// A copy, so that the dropped updates are freed.
+		kept = slices.Clone(kept[len(kept)-prefixBatchSize:])
+	}
+
+	// takeBatch left held empty, and only this flush fills it.
+	h.held = kept
+
+	for _, peer := range failedPeers {
+		if !slices.Contains(h.peersDown, peer) {
+			h.peersDown = append(h.peersDown, peer)
+		}
+	}
+
+	return toWrite
 }
 
 // writeBatch writes the collected route upserts, route deletions and prefix