Keep a peer-down until its route delete succeeds (closes #101)

When the delete of a peer's live routes after its session went down
failed, the down was dropped and the routes stayed. The prefix handler
now keeps such a peer, with the updates it sent after the down, for the
next flush, which retries the delete before it writes them; each
failure is logged. The delete runs in chunks of at most 10,000 routes
per table, taken in prefix order through the peer's key, each chunk in
its own transaction, so the database lock is released between chunks
and a failure undoes at most one chunk.

Model: opus-5-5

internal/routewatch/prefixhandler_internal_test.go

@@ -560,6 +560,124 @@ func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) {
 	})
 }
 
+// errTestPeerDelete is the error firstPeerDeleteFailsStore's first delete
+// returns.
+var errTestPeerDelete = errors.New("test: the delete of a peer's routes failed")
+
+// firstPeerDeleteFailsStore is a real database whose first delete of a peer's
+// live routes fails, deleting nothing. deletes counts the deletes asked for.
+type firstPeerDeleteFailsStore struct {
+	*database.Database
+
+	// whileFailing, if not nil, runs during the delete that fails.
+	whileFailing func()
+	deletes      int
+}
+
+// DeleteLiveRoutesOfPeer fails the first call and deletes the peer's routes on
+// every later one.
+func (s *firstPeerDeleteFailsStore) DeleteLiveRoutesOfPeer(
+	peerIP string,
+) (int, error) {
+	s.deletes++
+	if s.deletes > 1 {
+		return s.Database.DeleteLiveRoutesOfPeer(peerIP)
+	}
+
+	if s.whileFailing != nil {
+		s.whileFailing()
+	}
+
+	return 0, errTestPeerDelete
+}
+
+// TestPrefixHandlerRetriesAFailedDeleteOfAPeersRoutes runs, through the prefix
+// handler into a database whose first delete of a peer's routes fails, one flush
+// after another: a peer's announcement, then its session going down and a new
+// announcement from the peer, then nothing. The failed delete must be retried
+// on the next flush, and the announcement the peer sent after the down written
+// only after that retry, so it is kept. In the second case the peer's session
+// goes down again, and the peer announces another prefix, while the delete
+// fails: the announcement sent between the two downs must not be written.
+func TestPrefixHandlerRetriesAFailedDeleteOfAPeersRoutes(t *testing.T) {
+	t.Parallel()
+
+	const (
+		beforeDown      = "192.0.2.0/24"
+		afterDown       = "198.51.100.0/24"
+		afterSecondDown = "203.0.113.0/24"
+		maskLength      = 24
+	)
+
+	tests := []struct {
+		name string
+		// Handled while the delete fails, if not nil.
+		whileFailing []*ristypes.RISMessage
+		wantDeletes  int
+		// The peers left with a live route for each prefix.
+		wantPeers map[string][]string
+	}{
+		{
+			name:        "the delete fails once",
+			wantDeletes: 2,
+			wantPeers: map[string][]string{
+				beforeDown:      nil,
+				afterDown:       {testPeerIP},
+				afterSecondDown: nil,
+			},
+		},
+		{
+			name: "the session goes down again while the delete fails",
+			whileFailing: []*ristypes.RISMessage{
+				sessionDown(),
+				peerAnnouncement(testPeerIP, afterSecondDown),
+			},
+			// The retry and the delete for the second down.
+			wantDeletes: 3,
+			wantPeers: map[string][]string{
+				beforeDown:      nil,
+				afterDown:       nil,
+				afterSecondDown: {testPeerIP},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			db := newTestDatabase(t)
+			store := &firstPeerDeleteFailsStore{Database: db}
+
+			// Built without NewPrefixHandler's flush timer, so the test
+			// flushes itself.
+			h := &PrefixHandler{db: store, logger: logger.New()}
+			store.whileFailing = func() {
+				for _, msg := range tt.whileFailing {
+					h.HandleMessage(msg)
+				}
+			}
+
+			h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown))
+			h.flushBatch()
+			h.HandleMessage(sessionDown())
+			h.HandleMessage(peerAnnouncement(testPeerIP, afterDown))
+			h.flushBatch() // the delete fails
+			h.flushBatch()
+
+			if store.deletes != tt.wantDeletes {
+				t.Errorf("the store was asked for %d deletes of the peer's "+
+					"routes, want %d", store.deletes, tt.wantDeletes)
+			}
+
+			assertPeers(t, db, tt.wantPeers)
+			assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{
+				{MaskLength: maskLength, Count: 1},
+			})
+		})
+	}
+}
+
 // newTestDatabase opens a database in a new temporary directory. It is closed
 // when the test ends.
 func newTestDatabase(t *testing.T) *database.Database {