Remove a peer's live routes when its session goes down (closes #56)

RIS sends no withdrawals for the routes of a peer whose BGP session goes
down, so routewatch kept them.

The streamer now hands STATE messages to the prefix handler. They are never
dropped for backpressure: one that finds the queue full is held and queued
first once there is room, and the handler's other messages are dropped
meanwhile, so none overtakes it. Nothing waits. On a down, the prefix
handler drops the peer's pending updates and its next flush deletes all the
peer's live routes before the batch's other writes, keeping the in-memory
counts exact. The live route tables are unique on peer and prefix, which
also finds a peer's routes.

Model: opus-5-5

internal/routewatch/app_integration_test.go

@@ -364,6 +364,12 @@ func (m *mockStore) DeleteLiveRouteBatch(deletions []database.LiveRouteDeletion)
 	return nil
 }
 
+// DeleteLiveRoutesOfPeer mock implementation
+func (m *mockStore) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
+	// Simple mock - deletes nothing
+	return 0, nil
+}
+
 // GetOrCreateASNBatch mock implementation
 func (m *mockStore) GetOrCreateASNBatch(asns map[int]time.Time) error {
 	m.mu.Lock()

internal/routewatch/peerhandler.go

@@ -1,7 +1,7 @@
 package routewatch
 
 // peerhandler.go provides batched peer tracking functionality for BGP route monitoring.
-// It tracks BGP peers from all incoming RIS messages and maintains peer state in the database.
+// It tracks BGP peers from incoming UPDATE messages and maintains peer state in the database.
 
 import (
 	"strconv"
@@ -33,7 +33,7 @@ const (
 	peerBatchTimeout = 2 * time.Second
 )
 
-// PeerHandler tracks BGP peers from all message types using batched operations.
+// PeerHandler tracks BGP peers from UPDATE messages using batched operations.
 // It maintains a queue of peer updates and periodically flushes them to the database
 // in batches to improve performance. The handler deduplicates peer updates within
 // each batch, keeping only the most recent update for each peer IP address.
@@ -82,11 +82,11 @@ func NewPeerHandler(db database.Store, logger *logger.Logger) *PeerHandler {
 	return h
 }
 
-// WantsMessage returns true for all message types since peer information
-// is extracted from every RIS message regardless of type. This satisfies
-// the MessageHandler interface.
-func (h *PeerHandler) WantsMessage(_ string) bool {
-	return true
+// WantsMessage returns true for UPDATE messages only. A peer whose session
+// never comes up sends STATE messages and nothing else, so a STATE message is
+// not a sign of a peer that carries routes.
+func (h *PeerHandler) WantsMessage(messageType string) bool {
+	return messageType == ristypes.MessageTypeUpdate
 }
 
 // QueueCapacity returns the desired queue capacity for this handler.

internal/routewatch/prefixhandler.go

@@ -3,6 +3,7 @@ package routewatch
 import (
 	"net"
 	"net/netip"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -16,9 +17,10 @@ import (
 
 const (
 	// prefixHandlerQueueSize is the queue capacity for prefix tracking
-	// operations, about 4 seconds of feed at peak. The streamer drops rather
-	// than blocks when a queue is full, so this bounds memory. Batches still
-	// flush on a timer (prefixBatchTimeout).
+	// operations, about 4 seconds of feed at peak. The streamer drops updates
+	// rather than blocks when a queue is full (a STATE message is held until
+	// there is room), so this bounds memory. Batches still flush on a timer
+	// (prefixBatchTimeout).
 	prefixHandlerQueueSize = 20000
 
 	// prefixBatchSize is the number of prefix updates to batch together
@@ -40,17 +42,19 @@ const (
 )
 
 // PrefixHandler tracks BGP prefixes and maintains a live routing table in the database.
-// Routes are added on announcement and deleted on withdrawal.
+// Routes are added on announcement and deleted on withdrawal, and all routes of
+// a peer are deleted when its BGP session goes down.
 type PrefixHandler struct {
 	db      database.Store
 	logger  *logger.Logger
 	metrics *metrics.Tracker
 
-	// Batching. mu guards batch only: the flush goroutine takes the batch
-	// under mu and writes it after releasing mu, so HandleMessage waits for
-	// a database write only while batch is at prefixBatchMaxSize.
+	// Batching. mu guards batch and peersDown only: the flush goroutine takes
+	// both under mu and writes them after releasing mu, so HandleMessage
+	// waits for a database write only while batch is at prefixBatchMaxSize.
 	mu          sync.Mutex
 	batch       []prefixUpdate
+	peersDown   []string      // peers whose BGP session went down since the batch was taken
 	batchTaken  sync.Cond     // signalled when the flush goroutine takes batch; its L is &mu
 	batchFullCh chan struct{} // wakes the flush goroutine when batch is full
 	lastFlush   time.Time     // used only by the flush goroutine
@@ -93,8 +97,9 @@ func (h *PrefixHandler) SetMetricsTracker(metrics *metrics.Tracker) {
 
 // WantsMessage returns true if this handler wants to process messages of the given type
 func (h *PrefixHandler) WantsMessage(messageType string) bool {
-	// We only care about UPDATE messages for the routing table
-	return messageType == "UPDATE"
+	// UPDATE messages change routes; a STATE message can say a peer's
+	// session went down, which removes all of its routes.
+	return messageType == ristypes.MessageTypeUpdate || messageType == ristypes.MessageTypeState
 }
 
 // QueueCapacity returns the desired queue capacity for this handler
@@ -105,6 +110,14 @@ func (h *PrefixHandler) QueueCapacity() int {
 
 // HandleMessage processes a message to track prefix information
 func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
+	if msg.Type == ristypes.MessageTypeState {
+		if msg.State == ristypes.StateDown {
+			h.peerSessionDown(msg.Peer)
+		}
+
+		return
+	}
+
 	// Use the pre-parsed timestamp
 	timestamp := msg.ParsedTimestamp
 
@@ -170,6 +183,19 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
 	}
 }
 
+// peerSessionDown notes that the peer's BGP session went down, so that the next
+// flush deletes all of the peer's live routes before it writes the batch. The
+// peer's updates still in the batch came before the down, which replaces them,
+// so they are dropped; updates the peer sends later stay in the batch and are
+// written after the delete.
+func (h *PrefixHandler) peerSessionDown(peer string) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	h.batch = slices.DeleteFunc(h.batch, func(update prefixUpdate) bool { return update.peer == peer })
+	h.peersDown = append(h.peersDown, peer)
+}
+
 // flushLoop runs in a goroutine, the only one that writes the batch: it
 // flushes when the batch is full, periodically, and once more on Stop.
 func (h *PrefixHandler) flushLoop() {
@@ -194,33 +220,53 @@ func (h *PrefixHandler) flushLoop() {
 	}
 }
 
-// takeBatch returns the pending updates and leaves an empty batch in their
-// place for HandleMessage to fill.
-func (h *PrefixHandler) takeBatch() []prefixUpdate {
+// takeBatch returns the pending updates and the peers whose session went down
+// since the last flush, and leaves an empty batch and no peers in their place
+// for HandleMessage to fill.
+func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
-	if len(h.batch) == 0 {
-		return nil
+	if len(h.batch) == 0 && len(h.peersDown) == 0 {
+		return nil, nil
 	}
-	batch := h.batch
+	batch, peersDown := h.batch, h.peersDown
 	h.batch = make([]prefixUpdate, 0, prefixBatchSize)
+	h.peersDown = nil
 	h.batchTaken.Broadcast()
 
-	return batch
+	return batch, peersDown
 }
 
 // flushBatch takes the pending batch and writes it to the database, without
 // holding the mutex during the write.
 func (h *PrefixHandler) flushBatch() {
-	batch := h.takeBatch()
-	if len(batch) == 0 {
+	batch, peersDown := h.takeBatch()
+	if len(batch) == 0 && len(peersDown) == 0 {
 		return
 	}
 
 	startTime := time.Now()
 	batchSize := len(batch)
 
+	// The routes of a peer whose session went down are deleted first: the
+	// batch holds only the updates the peer sent after the down. Most downs
+	// come from peers that have no routes, so only a delete that removed
+	// routes is logged.
+	for _, peer := range peersDown {
+		deleted, err := h.db.DeleteLiveRoutesOfPeer(peer)
+		if err != nil {
+			h.logger.Error("Failed to delete the live routes of a peer whose session went down",
+				"peer", peer, "error", err)
+
+			continue
+		}
+		if deleted > 0 {
+			h.logger.Info("Deleted the live routes of a peer whose session went down",
+				"peer", peer, "routes", deleted)
+		}
+	}
+
 	// For each prefix and peer, keep only the last update the feed delivered.
 	// Updates from different peers for the same prefix are all kept. Arrival
 	// order decides, not the timestamp: timestamps are whole seconds, so
@@ -296,6 +342,7 @@ func (h *PrefixHandler) flushBatch() {
 	elapsed := time.Since(startTime)
 	h.logger.Debug("Flushed prefix batch",
 		"batch_size", batchSize,
+		"peers_down", len(peersDown),
 		"unique_prefixes", len(prefixesToUpdate),
 		"success", successCount,
 		"duration_ms", elapsed.Milliseconds(),

internal/routewatch/prefixhandler_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"slices"
+	"sync"
 	"testing"
 	"time"
 
@@ -288,6 +289,256 @@ func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) {
 	}
 }
 
+// TestStateMessagesGoToThePrefixHandlerNotThePeerHandler checks that STATE
+// messages go to the prefix handler, which acts on a down, and not to the peer
+// handler, which would record a peer whose session never comes up as active.
+func TestStateMessagesGoToThePrefixHandlerNotThePeerHandler(t *testing.T) {
+	if !(&PrefixHandler{}).WantsMessage(ristypes.MessageTypeState) {
+		t.Error("the prefix handler does not want STATE messages")
+	}
+	if (&PeerHandler{}).WantsMessage(ristypes.MessageTypeState) {
+		t.Error("the peer handler wants STATE messages")
+	}
+	if !(&PeerHandler{}).WantsMessage(ristypes.MessageTypeUpdate) {
+		t.Error("the peer handler does not want UPDATE messages")
+	}
+}
+
+// TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown runs
+// announcements from two peers through the prefix handler into a real
+// database, then a batch in which one of the peers announces a prefix and its
+// session goes down. The down deletes every live route of that peer, IPv4 and
+// IPv6, the one announced just before it included, and keeps the other peer's
+// routes for the same prefixes. An announcement the peer sends after the down,
+// in the same batch, is kept. The in-memory route counts and prefix
+// distribution must match the route tables.
+func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) {
+	const (
+		otherPeerIP = "2001:db8:ffff::2"
+		// Both peers announce the shared prefixes, only testPeerIP the others.
+		sharedV4   = "198.51.100.0/24"
+		sharedV6   = "2001:db8:1::/48"
+		onlyV4     = "203.0.113.0/24"
+		onlyV6     = "2001:db8:2::/48"
+		beforeDown = "192.0.2.0/24"
+		afterDown  = "2001:db8:3::/48"
+		// The mask lengths of the IPv4 and of the IPv6 prefixes.
+		maskLengthV4 = 24
+		maskLengthV6 = 48
+	)
+
+	tests := []struct {
+		name string
+		// Handled and flushed as one batch after both peers' announcements.
+		batch []*ristypes.RISMessage
+		// Whether testPeerIP is left with a live route for afterDown.
+		wantAfterDown bool
+	}{
+		{
+			name:  "the down ends the batch",
+			batch: []*ristypes.RISMessage{peerAnnouncement(testPeerIP, beforeDown), sessionDown(testPeerIP)},
+		},
+		{
+			name: "the peer announces a prefix after the down",
+			batch: []*ristypes.RISMessage{
+				peerAnnouncement(testPeerIP, beforeDown),
+				sessionDown(testPeerIP),
+				peerAnnouncement(testPeerIP, afterDown),
+			},
+			wantAfterDown: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
+			if err != nil {
+				t.Fatalf("failed to create database: %v", err)
+			}
+			defer func() { _ = db.Close() }()
+
+			// Built without NewPrefixHandler's flush timer, so the test flushes itself.
+			h := &PrefixHandler{db: db, logger: logger.New()}
+			h.HandleMessage(peerAnnouncement(testPeerIP, sharedV4, sharedV6, onlyV4, onlyV6))
+			h.HandleMessage(peerAnnouncement(otherPeerIP, sharedV4, sharedV6))
+			h.flushBatch()
+			for _, msg := range tt.batch {
+				h.HandleMessage(msg)
+			}
+			h.flushBatch()
+
+			// The peers left with a live route for each prefix.
+			wantPeers := map[string][]string{
+				sharedV4:   {otherPeerIP},
+				sharedV6:   {otherPeerIP},
+				onlyV4:     nil,
+				onlyV6:     nil,
+				beforeDown: nil,
+				afterDown:  nil,
+			}
+			wantRoutesV6 := 1
+			if tt.wantAfterDown {
+				wantPeers[afterDown] = []string{testPeerIP}
+				wantRoutesV6++
+			}
+
+			assertPeers(t, db, wantPeers)
+			assertCounts(t, db, ipv4Version, 1,
+				[]database.PrefixDistribution{{MaskLength: maskLengthV4, Count: 1}})
+			assertCounts(t, db, ipv6Version, wantRoutesV6,
+				[]database.PrefixDistribution{{MaskLength: maskLengthV6, Count: wantRoutesV6}})
+		})
+	}
+}
+
+// TestPrefixHandlerKeepsWhatAPeerAnnouncesAfterTheFlushOfItsDown runs, through
+// the prefix handler into a real database, one flush after another: a peer's
+// announcement, its session going down, a new announcement from the peer, and
+// another peer's announcement. The down deletes the peer's routes on the flush
+// that handles it and on no later flush, so the route the peer announced after
+// it is kept.
+func TestPrefixHandlerKeepsWhatAPeerAnnouncesAfterTheFlushOfItsDown(t *testing.T) {
+	const (
+		otherPeerIP = "2001:db8:ffff::2"
+		beforeDown  = "192.0.2.0/24"
+		afterDown   = "198.51.100.0/24"
+		otherPeers  = "203.0.113.0/24"
+		maskLength  = 24
+	)
+
+	db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	// Built without NewPrefixHandler's flush timer, so the test flushes itself.
+	h := &PrefixHandler{db: db, logger: logger.New()}
+	for _, msg := range []*ristypes.RISMessage{
+		peerAnnouncement(testPeerIP, beforeDown),
+		sessionDown(testPeerIP),
+		peerAnnouncement(testPeerIP, afterDown),
+		peerAnnouncement(otherPeerIP, otherPeers),
+	} {
+		h.HandleMessage(msg)
+		h.flushBatch()
+	}
+
+	assertPeers(t, db, map[string][]string{
+		beforeDown: nil,
+		afterDown:  {testPeerIP},
+		otherPeers: {otherPeerIP},
+	})
+	assertCounts(t, db, ipv4Version, 2, []database.PrefixDistribution{{MaskLength: maskLength, Count: 2}})
+}
+
+// firstWriteHeldStore is a real database whose first live route batch write
+// waits until the test closes release. writeStarted is closed when it starts.
+type firstWriteHeldStore struct {
+	*database.Database
+
+	writeStarted chan struct{}
+	release      chan struct{}
+	once         sync.Once
+}
+
+func (s *firstWriteHeldStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
+	s.once.Do(func() {
+		close(s.writeStarted)
+		<-s.release
+	})
+
+	return s.Database.UpsertLiveRouteBatch(routes)
+}
+
+// TestPrefixHandlerHandlesADownWhileAFlushWrites holds the write of a flush
+// that stores a peer's route and meanwhile hands the prefix handler the peer's
+// session going down, then a new announcement from the peer. The next flush
+// must delete the route the held write stored and keep the one announced after
+// the down.
+func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) {
+	const (
+		beforeDown = "192.0.2.0/24"
+		afterDown  = "198.51.100.0/24"
+		maskLength = 24
+	)
+
+	db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+	store := &firstWriteHeldStore{Database: db, writeStarted: make(chan struct{}), release: make(chan struct{})}
+
+	// Built without NewPrefixHandler's flush timer, so the test flushes itself.
+	h := &PrefixHandler{db: store, logger: logger.New()}
+	h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown))
+	flushed := make(chan struct{})
+	go func() {
+		h.flushBatch()
+		close(flushed)
+	}()
+	select {
+	case <-store.writeStarted:
+	case <-time.After(handleMessageTimeout):
+		t.Fatal("the flush did not start its write")
+	}
+
+	if !returnsInTime(func() {
+		h.HandleMessage(sessionDown(testPeerIP))
+		h.HandleMessage(peerAnnouncement(testPeerIP, afterDown))
+	}) {
+		t.Fatal("HandleMessage did not return while a database write was held")
+	}
+	close(store.release)
+	select {
+	case <-flushed:
+	case <-time.After(handleMessageTimeout):
+		t.Fatal("the flush did not finish after its write was let go")
+	}
+	h.flushBatch()
+
+	assertPeers(t, db, map[string][]string{beforeDown: nil, afterDown: {testPeerIP}})
+	assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{{MaskLength: maskLength, Count: 1}})
+}
+
+// peerAnnouncement returns an UPDATE message in which peer announces prefixes
+// with origin AS testASNB.
+func peerAnnouncement(peer string, prefixes ...string) *ristypes.RISMessage {
+	return &ristypes.RISMessage{
+		Type:            ristypes.MessageTypeUpdate,
+		Peer:            peer,
+		Path:            ristypes.ASPath{testASNA, testASNB},
+		ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
+		Announcements:   []ristypes.RISAnnouncement{{NextHop: peer, Prefixes: prefixes}},
+	}
+}
+
+// sessionDown returns the STATE message that says peer's BGP session went down.
+func sessionDown(peer string) *ristypes.RISMessage {
+	return &ristypes.RISMessage{Type: ristypes.MessageTypeState, Peer: peer, State: ristypes.StateDown}
+}
+
+// assertPeers checks, for each prefix in wantPeers, that the peers with a live
+// route for it are the ones listed, in peer IP order; nil means none.
+func assertPeers(t *testing.T, db *database.Database, wantPeers map[string][]string) {
+	t.Helper()
+
+	for prefix, want := range wantPeers {
+		routes, _, err := db.GetPrefixDetailsContext(context.Background(), prefix)
+		if err != nil && !errors.Is(err, database.ErrNoRoute) {
+			t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err)
+		}
+		var got []string
+		for _, route := range routes {
+			got = append(got, route.PeerIP)
+		}
+		if !slices.Equal(got, want) {
+			t.Errorf("peers with a live route for %s = %v, want %v", prefix, got, want)
+		}
+	}
+}
+
 // assertCounts checks that the live route count and prefix distribution of one
 // IP version are wantRoutes and wantDistribution, both as held in memory and as
 // read from the route table.