routewatch/internal/routewatch/prefixhandler.go

package routewatch

import (
	"context"
	"net"
	"net/netip"
	"slices"
	"strings"
	"sync"
	"time"

	"git.eeqj.de/sneak/routewatch/internal/database"
	"git.eeqj.de/sneak/routewatch/internal/logger"
	"git.eeqj.de/sneak/routewatch/internal/metrics"
	"git.eeqj.de/sneak/routewatch/internal/ristypes"
)

const (
	// prefixHandlerQueueSize is the queue capacity for prefix tracking
	// operations, about 4 seconds of feed at peak. The streamer drops updates
	// rather than blocks when a queue is full (a STATE message is held until
	// there is room), so this bounds memory. Batches still flush on a timer
	// (prefixBatchTimeout).
	prefixHandlerQueueSize = 20000

	// prefixBatchSize is the number of prefix updates to batch together
	prefixBatchSize = 25000

	// prefixBatchMaxSize is the most prefix updates the batch holds while the
	// flush goroutine writes the previous one: ten batches, about 60 MiB
	// (README "Memory"). At this size HandleMessage waits until the flush
	// goroutine takes the batch. On top of these, the handler holds at most
	// prefixBatchSize updates, about 6 MiB, that a flush kept for the peers
	// whose route delete failed (held).
	prefixBatchMaxSize = 250000

	// prefixBatchTimeout is the maximum time to wait before flushing a batch
	// DO NOT reduce this timeout - larger batches are more efficient
	prefixBatchTimeout = 1 * time.Second

	// IP version constants
	ipv4Version = 4
	ipv6Version = 6
)

// PrefixHandler tracks BGP prefixes and maintains a live routing table in the
// database. Routes are added on announcement and deleted on withdrawal or on an
// announcement whose AS path is empty or holds AS 0 anywhere, and all routes of
// a peer are deleted when its BGP session goes down.
type PrefixHandler struct {
	db      database.Store
	logger  *logger.Logger
	metrics *metrics.Tracker

	// Batching. mu guards batch, held, peersDown, writing and deleting only:
	// the flush goroutine takes them under mu and writes them after releasing
	// mu, so HandleMessage waits for a database write only while batch is at
	// prefixBatchMaxSize.
	mu    sync.Mutex
	batch []prefixUpdate
	// held holds the updates that the peers whose route delete failed in the
	// last flush sent after their session went down, for the next flush to
	// write after it retries the delete. They are not in batch, so they do
	// not count toward a full batch: a delete that keeps failing is retried
	// at the pace of the flush timer.
	held []prefixUpdate
	// peersDown holds, each once, the peers whose BGP session went down
	// since the batch was taken, and those whose route delete failed in the
	// last flush.
	peersDown []string
	// writing is how many updates the flush goroutine took and has not
	// finished writing.
	writing int
	// deleting is how many peers from peersDown the flush goroutine took and
	// has not yet deleted the routes of.
	deleting int
	// batchTaken is signalled when the flush goroutine takes batch; its L is
	// &mu.
	batchTaken  sync.Cond
	batchFullCh chan struct{} // wakes the flush goroutine when batch is full
	lastFlush   time.Time     // used only by the flush goroutine
	// stopCh hands Stop's context to the flush goroutine. It holds one, so
	// Stop does not wait for a write in progress to hand it over.
	stopCh chan context.Context
	done   chan struct{} // closed when the flush goroutine returns
}

type prefixUpdate struct {
	prefix      string
	originASN   int
	peer        string
	messageType string
	timestamp   time.Time
	path        []int
	// endsInSet says the path ends in an AS set of several members, so it
	// gives no origin.
	endsInSet bool
}

// NewPrefixHandler creates a new batched prefix tracking handler
func NewPrefixHandler(db database.Store, logger *logger.Logger) *PrefixHandler {
	h := &PrefixHandler{
		db:          db,
		logger:      logger,
		batch:       make([]prefixUpdate, 0, prefixBatchSize),
		batchFullCh: make(chan struct{}, 1),
		lastFlush:   time.Now(),
		stopCh:      make(chan context.Context, 1),
		done:        make(chan struct{}),
	}
	h.batchTaken.L = &h.mu

	// Start the flush timer goroutine
	go h.flushLoop()

	return h
}

// SetMetricsTracker sets the metrics tracker for recording route updates
func (h *PrefixHandler) SetMetricsTracker(metrics *metrics.Tracker) {
	h.metrics = metrics
}

// WantsMessage returns true if this handler wants to process messages of the
// given type
func (h *PrefixHandler) WantsMessage(messageType string) bool {
	// UPDATE messages change routes; a STATE message can say a peer's
	// session went down, which removes all of its routes.
	return messageType == ristypes.MessageTypeUpdate ||
		messageType == ristypes.MessageTypeState
}

// QueueCapacity returns the desired queue capacity for this handler
func (h *PrefixHandler) QueueCapacity() int {
	// Batching allows us to use a larger queue
	return prefixHandlerQueueSize
}

// HandleMessage processes a message to track prefix information
func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
	if msg.Type == ristypes.MessageTypeState {
		if msg.State == ristypes.StateDown {
			h.peerSessionDown(msg.Peer)
		}

		return
	}

	// Use the pre-parsed timestamp
	timestamp := msg.ParsedTimestamp

	originASN := msg.Path.Origin()

	// The stored and shown AS path lists the members of an AS set that ends
	// the path after the rest of it.
	path := slices.Concat(msg.Path.Sequence, msg.Path.Set)

	h.mu.Lock()
	defer h.mu.Unlock()

	// While the batch is at its maximum size, wait until the flush goroutine
	// takes it. Meanwhile the handler takes nothing off its queue, which
	// fills, and the streamer drops messages for it.
	for len(h.batch) >= prefixBatchMaxSize {
		h.batchTaken.Wait()
	}

	// Withdrawals are queued before announcements, because the batch keeps
	// each peer's last update for a prefix: a message that lists a prefix as
	// both withdrawn and announced keeps the route (RFC 4271 section 4.3).
	// A withdrawal names only the prefix and peer: the message's AS path, and
	// so its origin AS, belongs to the announced prefixes.
	for _, prefix := range msg.Withdrawals {
		h.batch = append(h.batch, prefixUpdate{
			prefix:      canonicalPrefix(prefix),
			peer:        msg.Peer,
			messageType: "withdrawal",
			timestamp:   timestamp,
		})
		// Record withdrawal in metrics
		if h.metrics != nil {
			h.metrics.RecordWithdrawal()
		}
	}

	for _, announcement := range msg.Announcements {
		for _, prefix := range announcement.Prefixes {
			h.batch = append(h.batch, prefixUpdate{
				prefix:      canonicalPrefix(prefix),
				originASN:   originASN,
				peer:        msg.Peer,
				messageType: "announcement",
				timestamp:   timestamp,
				path:        path,
				endsInSet:   len(msg.Path.Set) > 0,
			})
			// Record announcement in metrics
			if h.metrics != nil {
				h.metrics.RecordAnnouncement()
			}
		}
	}

	// A full batch wakes the flush goroutine. If it is writing, the wake-up
	// waits in batchFullCh and the batch keeps growing until the write ends
	// or the batch reaches prefixBatchMaxSize.
	if len(h.batch) >= prefixBatchSize {
		select {
		case h.batchFullCh <- struct{}{}:
		default: // already woken
		}
	}
}

// Stop stops the flush goroutine, which writes the updates and peer downs the
// handler holds before it returns, and waits for it until ctx ends. Once ctx
// has ended the flush goroutine skips that last write, and a write still
// running is lost when the process exits. A peer whose route delete fails in
// the last write is not retried either. Stop logs how many updates are not
// written and how many peer downs have not had their routes deleted.
func (h *PrefixHandler) Stop(ctx context.Context) {
	h.stopCh <- ctx

	select {
	case <-h.done:
	case <-ctx.Done():
	}

	h.mu.Lock()
	unwritten := h.writing + len(h.batch) + len(h.held)
	peersDown := h.deleting + len(h.peersDown)
	h.mu.Unlock()

	if unwritten == 0 && peersDown == 0 {
		return
	}

	if ctx.Err() != nil {
		h.logger.Error("Stop deadline passed before the prefix handler wrote "+
			"its updates", "updates", unwritten, "peers_down", peersDown)

		return
	}

	h.logger.Error("The prefix handler stopped without deleting the routes "+
		"of peers whose session went down", "updates", unwritten,
		"peers_down", peersDown)
}

// peerSessionDown notes that the peer's BGP session went down, so that the next
// flush deletes all of the peer's live routes before it writes the batch. The
// peer's updates still in the batch or held came before the down, which
// replaces them, so they are dropped; updates the peer sends later stay in the
// batch and are written after the delete. A peer already in peersDown is not
// added again: one delete removes all of its routes.
func (h *PrefixHandler) peerSessionDown(peer string) {
	h.mu.Lock()
	defer h.mu.Unlock()

	isPeers := func(update prefixUpdate) bool { return update.peer == peer }
	h.batch = slices.DeleteFunc(h.batch, isPeers)
	h.held = slices.DeleteFunc(h.held, isPeers)

	if !slices.Contains(h.peersDown, peer) {
		h.peersDown = append(h.peersDown, peer)
	}
}

// flushLoop runs in a goroutine, the only one that writes the batch: it
// flushes when the batch is full, periodically, and once more on Stop.
func (h *PrefixHandler) flushLoop() {
	defer close(h.done)

	ticker := time.NewTicker(prefixBatchTimeout)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			if time.Since(h.lastFlush) >= prefixBatchTimeout {
				h.flushBatch()
			}
		case <-h.batchFullCh:
			h.flushBatch()
		case ctx := <-h.stopCh:
			// Final flush, unless the stop deadline has passed
			if ctx.Err() == nil {
				h.flushBatch()
			}

			return
		}
	}
}

// takeBatch returns the pending updates, the held ones first, and the peers in
// peersDown, and leaves an empty batch, no held updates and no peers in their
// place for HandleMessage to fill. The updates count in writing until
// flushBatch has written them, and the peers in deleting until their routes
// are deleted.
func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) {
	h.mu.Lock()
	defer h.mu.Unlock()

	if len(h.held) == 0 && len(h.batch) == 0 && len(h.peersDown) == 0 {
		return nil, nil
	}

	batch, peersDown := h.batch, h.peersDown
	if len(h.held) > 0 {
		batch = slices.Concat(h.held, h.batch)
	}

	h.writing = len(batch)
	h.deleting = len(peersDown)
	h.held = nil
	h.batch = make([]prefixUpdate, 0, prefixBatchSize)
	h.peersDown = nil
	h.batchTaken.Broadcast()

	return batch, peersDown
}

// flushBatch takes the pending batch and writes it to the database, without
// holding the mutex during the write.
func (h *PrefixHandler) flushBatch() {
	batch, peersDown := h.takeBatch()
	if len(batch) == 0 && len(peersDown) == 0 {
		return
	}

	startTime := time.Now()
	batchSize := len(batch)

	// The routes of a peer whose session went down are deleted first: the
	// batch holds only the updates the peer sent after the down. A peer whose
	// delete failed is kept, with those updates, for the next flush to retry.
	failedPeers := h.deleteLiveRoutesOfPeers(peersDown)
	if len(failedPeers) > 0 {
		batch = h.keepForNextFlush(batch, failedPeers)
	}

	// For each prefix and peer, keep only the last update the feed delivered.
	// Updates from different peers for the same prefix are all kept. Arrival
	// order decides, not the timestamp: timestamps are whole seconds, so
	// updates within one second carry the same one.
	type prefixAndPeer struct{ prefix, peer string }

	lastUpdates := make(map[prefixAndPeer]prefixUpdate)

	// Each prefix the batch names, announced or withdrawn, with the earliest
	// timestamp among all of its updates, not only each peer's last one: the
	// prefix tables store only first_seen, written when the prefix is new and
	// never changed after.
	prefixesToUpdate := make(map[string]time.Time)

	for _, update := range batch {
		lastUpdates[prefixAndPeer{update.prefix, update.peer}] = update

		seen, exists := prefixesToUpdate[update.prefix]
		if !exists || update.timestamp.Before(seen) {
			prefixesToUpdate[update.prefix] = update.timestamp
		}
	}

	// Collect routes to upsert and delete
	var (
		routesToUpsert []*database.LiveRoute
		routesToDelete []database.LiveRouteDeletion
	)

	for _, update := range lastUpdates {
		// An announcement is stored when its path gives an origin, or when it
		// ends in an AS set of several members: that route is stored with
		// origin 0 (none). Any other update deletes the peer's route for the
		// prefix: a withdrawal, or an announcement whose path is empty or
		// holds AS 0 anywhere (the decoder empties such a path). Such an
		// announcement replaces the peer's route, as any announcement does,
		// with none (RFC 7607 treats AS 0 in a path as a withdrawal).
		if update.messageType == "announcement" &&
			(update.originASN > 0 || update.endsInSet) {
			// Create live route for batch upsert
			route := h.createLiveRoute(update)
			if route != nil {
				routesToUpsert = append(routesToUpsert, route)
			}
		} else {
			// Parse CIDR to get IP version
			_, ipVersion, err := parseCIDR(update.prefix)
			if err != nil {
				h.logger.Error("Failed to parse CIDR for route deletion",
					"prefix", update.prefix, "error", err)

				continue
			}

			// Create deletion record for batch delete
			routesToDelete = append(routesToDelete, database.LiveRouteDeletion{
				Prefix:    update.prefix,
				PeerIP:    update.peer,
				IPVersion: ipVersion,
			})
		}
	}

	successCount := h.writeBatch(routesToUpsert, routesToDelete, prefixesToUpdate)

	h.mu.Lock()
	h.writing = 0
	h.mu.Unlock()

	elapsed := time.Since(startTime)
	h.logger.Debug("Flushed prefix batch",
		"batch_size", batchSize,
		"peers_down", len(peersDown),
		"unique_prefixes", len(prefixesToUpdate),
		"success", successCount,
		"duration_ms", elapsed.Milliseconds(),
	)

	h.lastFlush = time.Now()
}

// deleteLiveRoutesOfPeers deletes all live routes of each of peers, the peers
// whose BGP session went down, and returns those whose delete failed. Most
// downs come from peers that have no routes, so only a delete that removed
// routes is logged. A peer whose delete succeeded no longer counts in deleting;
// a failed one counts until keepForNextFlush puts it back in peersDown.
func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) []string {
	var failed []string

	for _, peer := range peers {
		deleted, err := h.db.DeleteLiveRoutesOfPeer(peer)
		if err != nil {
			h.logger.Error(
				"Failed to delete the live routes of a peer whose session "+
					"went down; the next flush retries",
				"peer", peer, "routes_deleted", deleted, "error", err)

			failed = append(failed, peer)

			continue
		}

		h.mu.Lock()
		h.deleting--
		h.mu.Unlock()

		if deleted > 0 {
			h.logger.Info(
				"Deleted the live routes of a peer whose session went down",
				"peer", peer, "routes", deleted)
		}
	}

	return failed
}

// keepForNextFlush puts the peers whose route delete failed back in peersDown,
// and their updates from batch in held, so that the next flush retries each
// delete before it writes the peer's updates; written now, they would be
// deleted by the retry. The updates of a peer whose session went down again
// since the batch was taken came before that down, so they are dropped, as
// peerSessionDown drops them, and the peer is already in peersDown. It returns
// the updates of the other peers, for this flush to write.
//
// While a delete keeps failing, the held updates would grow with every flush,
// so only the last prefixBatchSize of them are kept and the older ones are
// dropped. This never leaves in place a route the peer withdrew: a prefix with
// a kept update keeps its last one, and the retry deletes all of the peer's
// routes, so a prefix whose updates were all dropped has no route from the peer
// until the peer announces it again.
func (h *PrefixHandler) keepForNextFlush(
	batch []prefixUpdate, failedPeers []string,
) []prefixUpdate {
	var kept, toWrite []prefixUpdate

	for _, update := range batch {
		if slices.Contains(failedPeers, update.peer) {
			kept = append(kept, update)
		} else {
			toWrite = append(toWrite, update)
		}
	}

	h.mu.Lock()
	defer h.mu.Unlock()

	kept = slices.DeleteFunc(kept, func(update prefixUpdate) bool {
		return slices.Contains(h.peersDown, update.peer)
	})

	if len(kept) > prefixBatchSize {
		h.logger.Error(
			"Dropped the oldest updates of peers whose route delete failed",
			"peers", failedPeers, "dropped", len(kept)-prefixBatchSize)

		// A copy, so that the dropped updates are freed.
		kept = slices.Clone(kept[len(kept)-prefixBatchSize:])
	}

	// takeBatch left held empty, and only this flush fills it. The kept
	// updates now count in held, not in writing, and the failed peers in
	// peersDown, not in deleting.
	h.held = kept
	h.writing = len(toWrite)
	h.deleting = 0

	for _, peer := range failedPeers {
		if !slices.Contains(h.peersDown, peer) {
			h.peersDown = append(h.peersDown, peer)
		}
	}

	return toWrite
}

// writeBatch writes the collected route upserts, route deletions and prefix
// updates to the database and returns how many routes it wrote.
func (h *PrefixHandler) writeBatch(
	routesToUpsert []*database.LiveRoute,
	routesToDelete []database.LiveRouteDeletion,
	prefixesToUpdate map[string]time.Time,
) int {
	successCount := 0

	if len(routesToUpsert) > 0 {
		err := h.db.UpsertLiveRouteBatch(routesToUpsert)
		if err != nil {
			h.logger.Error("Failed to upsert route batch",
				"error", err, "count", len(routesToUpsert))
		} else {
			successCount += len(routesToUpsert)
		}
	}

	if len(routesToDelete) > 0 {
		err := h.db.DeleteLiveRouteBatch(routesToDelete)
		if err != nil {
			h.logger.Error("Failed to delete route batch",
				"error", err, "count", len(routesToDelete))
		} else {
			successCount += len(routesToDelete)
		}
	}

	// Update prefix tables
	if len(prefixesToUpdate) > 0 {
		err := h.db.UpdatePrefixesBatch(prefixesToUpdate)
		if err != nil {
			h.logger.Error("Failed to update prefix batch",
				"error", err, "count", len(prefixesToUpdate))
		}
	}

	return successCount
}

// canonicalPrefix returns prefix in the text form net/netip prints, the form
// the IP lookup builds when it looks a prefix up. The feed sends IPv6
// announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed
// ("2001:db8:0:0:0:0:0:0/32"); stored as received, a withdrawal would not match
// the route its announcement stored. A prefix that does not parse is returned
// unchanged, and the batch flush reports it.
func canonicalPrefix(prefix string) string {
	p, err := netip.ParsePrefix(prefix)
	if err != nil {
		return prefix
	}

	return p.Masked().String()
}

// parseCIDR returns the mask length and the IP version of a prefix string
func parseCIDR(prefix string) (int, int, error) {
	_, ipNet, err := net.ParseCIDR(prefix)
	if err != nil {
		return 0, 0, err
	}

	ones, _ := ipNet.Mask.Size()
	if strings.Contains(prefix, ":") {
		return ones, ipv6Version, nil
	}

	return ones, ipv4Version, nil
}

// createLiveRoute creates a LiveRoute from a prefix update
func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute {
	// Parse CIDR to get mask length
	maskLength, ipVersion, err := parseCIDR(update.prefix)
	if err != nil {
		h.logger.Error("Failed to parse CIDR",
			"prefix", update.prefix,
			"error", err,
		)

		return nil
	}

	// Track route update metrics
	if h.metrics != nil {
		if ipVersion == ipv4Version {
			h.metrics.RecordIPv4Update()
		} else {
			h.metrics.RecordIPv6Update()
		}
	}

	// Create live route record
	liveRoute := &database.LiveRoute{
		Prefix:      update.prefix,
		MaskLength:  maskLength,
		IPVersion:   ipVersion,
		OriginASN:   update.originASN,
		PeerIP:      update.peer,
		ASPath:      update.path,
		NextHop:     update.peer, // Using peer as next hop
		LastUpdated: update.timestamp,
	}

	return liveRoute
}