Keep a peer-down until its route delete succeeds (closes #101)

When the delete of a peer's live routes after its session went down
failed, the down was dropped and the routes stayed. The prefix handler
now keeps such a peer, once, with the updates it sent after the down,
for the next flush, which retries the delete before it writes them; each
failure is logged. At most the last 25,000 of those updates are kept,
outside the batch, so they do not start a flush. The delete runs in
chunks of at most 10,000 routes per table, taken in prefix order
through the peer's key, each chunk in its own transaction, so the
database lock is released between chunks and a failure undoes at most
one chunk.

Model: opus-5-5

internal/database/counts_internal_test.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"context"
+	"fmt"
 	"slices"
 	"sync"
 	"testing"
@@ -494,6 +495,124 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) {
 	checkOneLiveRoutePerIPVersion(t, db)
 }
 
+// TestDeleteLiveRoutesOfPeerDeletesInChunks gives a peer one IPv4 route more
+// than a chunk holds, two of them for prefixes another peer also has. One chunk
+// deletes peerRouteChunkSize of the peer's routes and commits, leaving the last
+// one. The routes are written again, and DeleteLiveRoutesOfPeer deletes all of
+// them, a chunk and then the last one. After each step the route count and
+// prefix distribution held in memory and those read from the route table are
+// the ones wanted.
+func TestDeleteLiveRoutesOfPeerDeletesInChunks(t *testing.T) {
+	t.Parallel()
+
+	const (
+		peerIP      = "192.0.2.1"
+		otherPeerIP = "192.0.2.2"
+		// How many of the peer's prefixes the other peer also has, the first
+		// ones in prefix order.
+		sharedPrefixes = 2
+	)
+
+	db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+	peerRoutes := make([]*LiveRoute, 0, peerRouteChunkSize+1)
+	otherPeerRoutes := make([]*LiveRoute, 0, sharedPrefixes)
+
+	for i := range peerRouteChunkSize + 1 {
+		prefix := fmt.Sprintf("10.%d.%d.0/24", i/256, i%256)
+
+		route := mkV4Route(t, prefix, 64500, ts)
+		route.PeerIP = peerIP
+		peerRoutes = append(peerRoutes, route)
+
+		if i < sharedPrefixes {
+			route := mkV4Route(t, prefix, 64500, ts)
+			route.PeerIP = otherPeerIP
+			otherPeerRoutes = append(otherPeerRoutes, route)
+		}
+	}
+
+	err = db.UpsertLiveRouteBatch(slices.Concat(peerRoutes, otherPeerRoutes))
+	if err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+
+	deletedV4, deletedV6, err := db.deletePeerRouteChunk(peerIP)
+	if err != nil {
+		t.Fatalf("deletePeerRouteChunk: %v", err)
+	}
+
+	if deletedV4 != peerRouteChunkSize || deletedV6 != 0 {
+		t.Errorf("deletePeerRouteChunk deleted (%d, %d) routes, want (%d, 0)",
+			deletedV4, deletedV6, peerRouteChunkSize)
+	}
+
+	// The shared prefixes went with the chunk, so each route left has a
+	// prefix of its own.
+	checkV4RoutesOfOwnPrefixes(t, db, "after one chunk", 1+sharedPrefixes)
+
+	err = db.UpsertLiveRouteBatch(peerRoutes)
+	if err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+
+	deleted, err := db.DeleteLiveRoutesOfPeer(peerIP)
+	if err != nil {
+		t.Fatalf("DeleteLiveRoutesOfPeer: %v", err)
+	}
+
+	if deleted != len(peerRoutes) {
+		t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d",
+			deleted, len(peerRoutes))
+	}
+
+	checkV4RoutesOfOwnPrefixes(t, db, "after DeleteLiveRoutesOfPeer",
+		sharedPrefixes)
+}
+
+// checkV4RoutesOfOwnPrefixes checks that the database has wantRoutes IPv4 live
+// routes, each for a /24 no other live route has, both as counted in memory and
+// as read from the route table. when says which step of the test checks.
+func checkV4RoutesOfOwnPrefixes(
+	t *testing.T, db *Database, when string, wantRoutes int,
+) {
+	t.Helper()
+
+	ctx := context.Background()
+
+	stats, err := db.GetStatsContext(ctx)
+	if err != nil {
+		t.Fatalf("GetStatsContext: %v", err)
+	}
+
+	tableRoutes, _, err := db.GetLiveRouteCountsContext(ctx)
+	if err != nil {
+		t.Fatalf("GetLiveRouteCountsContext: %v", err)
+	}
+
+	if stats.IPv4Routes != wantRoutes || tableRoutes != wantRoutes {
+		t.Errorf("%s: IPv4 live routes: %d in memory, %d in the table, want %d",
+			when, stats.IPv4Routes, tableRoutes, wantRoutes)
+	}
+
+	queryDistribution, _, err := db.GetPrefixDistributionContext(ctx)
+	if err != nil {
+		t.Fatalf("GetPrefixDistributionContext: %v", err)
+	}
+
+	want := []PrefixDistribution{{MaskLength: 24, Count: wantRoutes}}
+	assertDistribution(t, when+": IPv4 distribution",
+		stats.IPv4PrefixDistribution, want)
+	assertDistribution(t, when+": IPv4 distribution query",
+		queryDistribution, want)
+}
+
 // checkOneLiveRoutePerIPVersion checks that the database has one IPv4 live
 // route, for a /24, and one IPv6 live route, for a /32, both as counted in
 // memory and as read from the route tables.