Keep a peer-down until its route delete succeeds (closes #101)

When the delete of a peer's live routes after its session went down
failed, the down was dropped and the routes stayed. The prefix handler
now keeps such a peer, with the updates it sent after the down, for the
next flush, which retries the delete before it writes them; each
failure is logged. While the delete keeps failing, only the last 25,000
of those updates are kept, and the older ones are dropped and logged.
The delete runs in chunks of at most 10,000 routes per table, taken in
prefix order through the peer's key, each chunk in its own transaction,
so the database lock is released between chunks and a failure undoes
at most one chunk.

Model: opus-5-5

internal/database/counts_internal_test.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"context"
+	"fmt"
 	"slices"
 	"sync"
 	"testing"
@@ -494,6 +495,124 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) {
 	checkOneLiveRoutePerIPVersion(t, db)
 }
 
+// TestDeleteLiveRoutesOfPeerDeletesInChunks gives a peer one IPv4 route more
+// than a chunk holds, two of them for prefixes another peer also has. One chunk
+// deletes peerRouteChunkSize of the peer's routes and commits, leaving the last
+// one. The routes are written again, and DeleteLiveRoutesOfPeer deletes all of
+// them, a chunk and then the last one. After each step the route count and
+// prefix distribution held in memory and those read from the route table are
+// the ones wanted.
+func TestDeleteLiveRoutesOfPeerDeletesInChunks(t *testing.T) {
+	t.Parallel()
+
+	const (
+		peerIP      = "192.0.2.1"
+		otherPeerIP = "192.0.2.2"
+		// How many of the peer's prefixes the other peer also has, the first
+		// ones in prefix order.
+		sharedPrefixes = 2
+	)
+
+	db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+	peerRoutes := make([]*LiveRoute, 0, peerRouteChunkSize+1)
+	otherPeerRoutes := make([]*LiveRoute, 0, sharedPrefixes)
+
+	for i := range peerRouteChunkSize + 1 {
+		prefix := fmt.Sprintf("10.%d.%d.0/24", i/256, i%256)
+
+		route := mkV4Route(t, prefix, 64500, ts)
+		route.PeerIP = peerIP
+		peerRoutes = append(peerRoutes, route)
+
+		if i < sharedPrefixes {
+			route := mkV4Route(t, prefix, 64500, ts)
+			route.PeerIP = otherPeerIP
+			otherPeerRoutes = append(otherPeerRoutes, route)
+		}
+	}
+
+	err = db.UpsertLiveRouteBatch(slices.Concat(peerRoutes, otherPeerRoutes))
+	if err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+
+	deletedV4, deletedV6, err := db.deletePeerRouteChunk(peerIP)
+	if err != nil {
+		t.Fatalf("deletePeerRouteChunk: %v", err)
+	}
+
+	if deletedV4 != peerRouteChunkSize || deletedV6 != 0 {
+		t.Errorf("deletePeerRouteChunk deleted (%d, %d) routes, want (%d, 0)",
+			deletedV4, deletedV6, peerRouteChunkSize)
+	}
+
+	// The shared prefixes went with the chunk, so each route left has a
+	// prefix of its own.
+	checkV4RoutesOfOwnPrefixes(t, db, "after one chunk", 1+sharedPrefixes)
+
+	err = db.UpsertLiveRouteBatch(peerRoutes)
+	if err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+
+	deleted, err := db.DeleteLiveRoutesOfPeer(peerIP)
+	if err != nil {
+		t.Fatalf("DeleteLiveRoutesOfPeer: %v", err)
+	}
+
+	if deleted != len(peerRoutes) {
+		t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d",
+			deleted, len(peerRoutes))
+	}
+
+	checkV4RoutesOfOwnPrefixes(t, db, "after DeleteLiveRoutesOfPeer",
+		sharedPrefixes)
+}
+
+// checkV4RoutesOfOwnPrefixes checks that the database has wantRoutes IPv4 live
+// routes, each for a /24 no other live route has, both as counted in memory and
+// as read from the route table. when says which step of the test checks.
+func checkV4RoutesOfOwnPrefixes(
+	t *testing.T, db *Database, when string, wantRoutes int,
+) {
+	t.Helper()
+
+	ctx := context.Background()
+
+	stats, err := db.GetStatsContext(ctx)
+	if err != nil {
+		t.Fatalf("GetStatsContext: %v", err)
+	}
+
+	tableRoutes, _, err := db.GetLiveRouteCountsContext(ctx)
+	if err != nil {
+		t.Fatalf("GetLiveRouteCountsContext: %v", err)
+	}
+
+	if stats.IPv4Routes != wantRoutes || tableRoutes != wantRoutes {
+		t.Errorf("%s: IPv4 live routes: %d in memory, %d in the table, want %d",
+			when, stats.IPv4Routes, tableRoutes, wantRoutes)
+	}
+
+	queryDistribution, _, err := db.GetPrefixDistributionContext(ctx)
+	if err != nil {
+		t.Fatalf("GetPrefixDistributionContext: %v", err)
+	}
+
+	want := []PrefixDistribution{{MaskLength: 24, Count: wantRoutes}}
+	assertDistribution(t, when+": IPv4 distribution",
+		stats.IPv4PrefixDistribution, want)
+	assertDistribution(t, when+": IPv4 distribution query",
+		queryDistribution, want)
+}
+
 // checkOneLiveRoutePerIPVersion checks that the database has one IPv4 live
 // route, for a /24, and one IPv6 live route, for a /32, both as counted in
 // memory and as read from the route tables.