Write the routing table every second and read it at startup (part of #122)

The prefix handler applies each update to the routing table in memory as
it handles it, sorted into announcements and withdrawals as for the live
route tables. Each flush first writes at most 20,000 of the table's
changed pairs, with every new peer; the rest follow in the next flushes.
A failed write puts its changes and new peers back in the table. Stop
makes one last flush, and logs the changed pairs left.

Startup reads the table before the streamer connects, and logs its pairs,
peers and time. The per-peer counts are a slice, so the read takes about
a third less time. The health check's start period is 60 seconds, so a
full table's read fits before upaas looks.

Model: opus-5-5

internal/routetable/routetable.go

@@ -7,6 +7,7 @@ package routetable
 
 import (
 	"cmp"
+	"maps"
 	"net/netip"
 	"slices"
 	"sync"
@@ -102,11 +103,15 @@ type Table struct {
 	peerNumbers    map[string]uint
 	peerAddresses  map[uint]string
 	nextPeerNumber uint
-	// peerPrefixes counts the prefixes each peer carries, by its number.
-	peerPrefixes map[uint]int
+	// peerPrefixes[n] counts the prefixes the peer numbered n carries, for
+	// each n below nextPeerNumber. It is a slice, not a map, because Load
+	// counts once for every peer of every pair it loads: with a map, reading
+	// a full table at startup took several times as long.
+	peerPrefixes []int
 
-	// newPeers and changed are what TakeChanges hands over next: the peers
-	// numbered, and the prefix and origin AS pairs changed, since it last ran.
+	// newPeers and changed are what TakeChanges hands over: the peers
+	// numbered since it last ran, and the prefix and origin AS pairs changed
+	// since it last took them.
 	newPeers map[string]uint
 	changed  map[pair]struct{}
 
@@ -127,7 +132,6 @@ func New(params Params) *Table {
 		byOrigin:      make(map[int]map[netip.Prefix]*origin),
 		peerNumbers:   make(map[string]uint, len(params.PeerNumbers)),
 		peerAddresses: make(map[uint]string, len(params.PeerNumbers)),
-		peerPrefixes:  make(map[uint]int),
 		newPeers:      make(map[string]uint),
 		changed:       make(map[pair]struct{}),
 		keptTimes: btree.NewG(keptTimesDegree, func(a, b keptTime) bool {
@@ -141,6 +145,8 @@ func New(params Params) *Table {
 		t.nextPeerNumber = max(t.nextPeerNumber, number+1)
 	}
 
+	t.peerPrefixes = make([]int, t.nextPeerNumber)
+
 	return t
 }
 
@@ -240,13 +246,14 @@ func (t *Table) Load(change Change) {
 	o := t.findOrAddOrigin(prefix, change.OriginASN)
 	wasWithoutKeptRoute := o.withoutKeptRoute()
 
-	for peer := range change.Peers.EachSet() {
-		if !o.peers.Test(peer) {
-			o.peers.Set(peer)
+	// The peers not yet in the peer set join it in one union, which takes
+	// less time than setting their bits one at a time.
+	joining := change.Peers.Difference(&o.peers)
+	o.peers.InPlaceUnion(joining)
+	t.familyOf(prefix.Addr()).routes += int(joining.Count())
 
-			t.familyOf(prefix.Addr()).routes++
-			t.peerPrefixes[peer]++
-		}
+	for peer := range joining.EachSet() {
+		t.peerPrefixes[peer]++
 	}
 
 	// Oldest first, as they were announced, so that two routes with the same
@@ -265,16 +272,29 @@ func (t *Table) Load(change Change) {
 	t.countOriginWithoutKeptRoute(o, wasWithoutKeptRoute)
 }
 
-// TakeChanges returns the prefix and origin AS pairs that changed since it last
-// ran, in prefix order, each with its current content. It also returns the
-// peers numbered since then, by address. It then clears both records.
-func (t *Table) TakeChanges() ([]Change, map[string]uint) {
+// TakeChanges returns at most maxChanges of the prefix and origin AS pairs that
+// changed since they were last taken, in prefix order, each with its current
+// content. The other changed pairs wait for a later call. It also returns every
+// peer numbered since it last ran, by address, so a pair taken later names only
+// peers already returned. It then forgets what it returned.
+func (t *Table) TakeChanges(maxChanges int) ([]Change, map[string]uint) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	changes := make([]Change, 0, len(t.changed))
+	changes := make([]Change, 0, min(maxChanges, len(t.changed)))
 	for key := range t.changed {
+		if len(changes) == maxChanges {
+			break
+		}
+
 		changes = append(changes, t.currentContent(key))
+		delete(t.changed, key)
+	}
+
+	// A map keeps the memory of its largest size, so an emptied one is
+	// replaced.
+	if len(t.changed) == 0 {
+		t.changed = make(map[pair]struct{})
 	}
 
 	slices.SortFunc(changes, func(a, b Change) int {
@@ -284,11 +304,35 @@ func (t *Table) TakeChanges() ([]Change, map[string]uint) {
 
 	newPeers := t.newPeers
 	t.newPeers = make(map[string]uint)
-	t.changed = make(map[pair]struct{})
 
 	return changes, newPeers
 }
 
+// PutBackChanges records again what one TakeChanges call returned, for a
+// caller that could not write it. The next TakeChanges returns those peers
+// again, and those pairs wait to be taken again like any changed pair. Each is
+// then taken with its content at that time, so a later change of a pair
+// replaces the earlier.
+func (t *Table) PutBackChanges(changes []Change, newPeers map[string]uint) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	for _, change := range changes {
+		t.changed[pair{prefix: change.Prefix, asn: change.OriginASN}] = struct{}{}
+	}
+
+	maps.Copy(t.newPeers, newPeers)
+}
+
+// ChangeCount returns how many changed prefix and origin AS pairs wait for
+// TakeChanges.
+func (t *Table) ChangeCount() int {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return len(t.changed)
+}
+
 // numberPeer returns the peer's number, and gives it the next one the first
 // time the table sees it.
 func (t *Table) numberPeer(address string) uint {
@@ -301,6 +345,7 @@ func (t *Table) numberPeer(address string) uint {
 	t.nextPeerNumber++
 	t.peerNumbers[address] = number
 	t.peerAddresses[number] = address
+	t.peerPrefixes = append(t.peerPrefixes, 0)
 	t.newPeers[address] = number
 
 	return number