Keep one entry per AS in the AS handler's batch (closes #217)

HandleMessage added one entry to the batch for every AS of every path,
and waited for the write in progress once the batch held 300,000. In a
burst of the feed its queue then filled, and the streamer dropped
messages.

The batch is now a map from AS number to the earliest and the latest
time of its updates. It holds one entry per AS, so it no longer grows
with the feed's rate, and HandleMessage never waits for a write.
asnBatchSize now counts ASes. Stop logs the ASes it did not write under
"asns".

Model: opus-5-5

README.md

@@ -429,10 +429,12 @@ the routing table, and everything else from the database.
     - Parses BGP UPDATE messages from JSON stream
     - Dispatches to registered handlers based on message type
     - Each handler has its own queue with backpressure handling
-    - Handlers batch writes for efficiency: the AS, peer and prefix handlers
+    - Handlers batch writes for efficiency. The AS, peer and prefix handlers
-      write 30,000, 10,000 and 25,000 updates at a time, or after 2, 2 and 1
+      write a batch when it reaches a size, or after a time:
-      seconds; the peering handler writes the peerings it has collected every 30
+        - the AS handler at 30,000 ASes, or after 2 seconds;
-      seconds
+        - the peer handler at 10,000 updates, or after 2 seconds;
+        - the prefix handler at 25,000 updates, or after 1 second.
+    - The peering handler writes the peerings it has collected every 30 seconds
 
 4. **Handler Details**
     - **ASHandler**: Tracks all ASNs seen in AS paths
@@ -632,16 +634,18 @@ What happens at each limit:
   `STATE` message, which it holds, in feed order, until there is room: a few
   hundred bytes each, about ten a second.
 - Handler batches: while the AS, peer or prefix handler writes a batch to the
-  database, the updates that arrive collect in its next batch. That batch stops
+  database, the updates that arrive collect in its next batch.
-  at ten times the batch size:
+    - The AS handler's batch holds one entry per AS, with the earliest and the
-    - 300,000 updates for the AS handler (`asnBatchMaxSize`, about 10 MiB);
+      latest time of its updates. About 80,000 ASes appear in the feed, so the
-    - 100,000 for the peer handler (`peerBatchMaxSize`, about 10 MiB);
+      batch takes at most about 10 MiB, however fast the feed runs.
-    - 250,000 for the prefix handler (`prefixBatchMaxSize`, about 15 MiB).
+    - The peer and prefix handlers' batches stop at ten times their batch size:
+      100,000 updates for the peer handler (`peerBatchMaxSize`, about 10 MiB),
+      and 250,000 for the prefix handler (`prefixBatchMaxSize`, about 15 MiB).
+      At that size the handler takes nothing more off its queue until the write
+      ends. The queue then fills, and the streamer drops messages as above.
 
-    At that size the handler takes nothing more off its queue until the write
+    With the batches being written, the three handlers' batches take at most
-    ends. The queue then fills, and the streamer drops messages as above. With
+    about 0.1 GiB of the Go heap.
-    the batches being written, the three handlers' batches take at most about
-    0.1 GiB of the Go heap.
 
 With `DEBUG=routewatch` the daemon logs a `System stats` line every 60 seconds
 with the goroutine count and Go memory figures.

TODO.md

@@ -35,6 +35,11 @@ tested.
 
 # Completed Steps
 
+- 2026-11-02: the AS handler never waits for a database write. Its batch holds
+  one entry per AS, with the earliest and the latest time of its updates, so it
+  no longer grows with the feed's rate. It used to hold one entry for each AS of
+  each path and to wait once it held 300,000. In a burst of the feed its queue
+  then filled, and the streamer dropped messages for it (closes #217)
 - 2026-10-27: the tests of `internal/routewatch` no longer fail on a loaded
   host. Five of them gave `HandleMessage` two seconds to apply a full prefix
   batch, which under the race detector can take longer. A test now waits up to

internal/routewatch/app_integration_internal_test.go

@@ -644,9 +644,8 @@ func TestShutdownDoesNotWaitForABlockedWrite(t *testing.T) {
 	}
 
 	rw.dbHandler = NewASHandler(store, log)
-	fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
 	startBlockedWrite(t, blocking, rw.dbHandler,
-		[]*ristypes.RISMessage{{Path: fill}})
+		[]*ristypes.RISMessage{{Path: pathOfASes(asnBatchSize)}})
 
 	ctx, cancel := context.WithTimeout(t.Context(), deadline)
 	defer cancel()

internal/routewatch/ashandler.go

@@ -18,43 +18,34 @@ const (
 	// (asnBatchTimeout), so a queue smaller than asnBatchSize is fine.
 	asHandlerQueueSize = 20000
 
-	// asnBatchSize is the number of ASN operations to batch together
+	// asnBatchSize is the number of ASes in a full batch, which wakes the
+	// flush goroutine to write it.
 	asnBatchSize = 30000
 
-	// asnBatchMaxSize is the most ASN operations the batch holds while the
-	// flush goroutine writes the previous one: ten batches, about 10 MiB
-	// (README "Memory"). At this size HandleMessage waits until the flush
-	// goroutine takes the batch.
-	asnBatchMaxSize = 300000
-
 	// asnBatchTimeout is the maximum time to wait before flushing a batch
 	// DO NOT reduce this timeout - larger batches are more efficient
 	asnBatchTimeout = 2 * time.Second
 )
 
-// ASHandler processes Autonomous System Number (ASN) information extracted from
+// ASHandler records every AS in the AS paths of BGP UPDATE messages. It
-// BGP UPDATE messages. It uses batched database operations to efficiently store
+// collects, for each AS, the earliest and the latest time of its updates in a
-// ASN data, collecting operations into batches that are flushed either when the
+// batch. The flush goroutine writes the batch to the database when it holds
-// batch reaches a size threshold or after a timeout period.
+// asnBatchSize ASes, and every asnBatchTimeout.
 type ASHandler struct {
 	db     database.Store
 	logger *logger.Logger
 
-	// batchMaxSize is asnBatchMaxSize. Only HandleMessage reads it, so a test
-	// may lower it before it hands the handler a message.
-	batchMaxSize int
-
 	// Batching. mu guards batch and writing only: the flush goroutine takes
 	// the batch under mu and writes it after releasing mu, so HandleMessage
-	// waits for a database write only while batch is at batchMaxSize.
+	// never waits for a database write.
-	mu    sync.Mutex
+	mu sync.Mutex
-	batch []asnOp
+	// batch holds the earliest and the latest time of the updates for each AS
-	// writing is how many operations the flush goroutine took from batch
+	// since the flush goroutine last took it. It holds one entry per AS,
-	// and has not finished writing.
+	// however many updates arrive.
-	writing int
+	batch map[int]database.SeenTimes
-	// batchTaken is signalled when the flush goroutine takes batch; its L is
+	// writing is how many ASes the flush goroutine took from batch and has
-	// &mu.
+	// not finished writing.
-	batchTaken  sync.Cond
+	writing     int
 	batchFullCh chan struct{} // wakes the flush goroutine when batch is full
 	// stopCh hands Stop's context to the flush goroutine. It holds one, so
 	// Stop does not wait for a write in progress to hand it over.
@@ -62,16 +53,11 @@ type ASHandler struct {
 	done   chan struct{} // closed when the flush goroutine returns
 }
 
-type asnOp struct {
-	number    int
-	timestamp time.Time
-}
-
 // NewASHandler creates and returns a new ASHandler instance. It initializes
 // the batching system and starts a background goroutine that periodically
-// flushes accumulated ASN operations to the database. The caller must call
+// writes the collected ASes to the database. The caller must call Stop when
-// Stop when finished to ensure all pending operations are flushed and the
+// finished to ensure all pending ASes are written and the background goroutine
-// background goroutine is terminated.
+// is terminated.
 func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
 	return newASHandler(db, logger, asnBatchTimeout)
 }
@@ -83,15 +69,13 @@ func newASHandler(
 	db database.Store, logger *logger.Logger, batchTimeout time.Duration,
 ) *ASHandler {
 	h := &ASHandler{
-		db:           db,
+		db:          db,
-		logger:       logger,
+		logger:      logger,
-		batchMaxSize: asnBatchMaxSize,
+		batch:       make(map[int]database.SeenTimes),
-		batch:        make([]asnOp, 0, asnBatchSize),
+		batchFullCh: make(chan struct{}, 1),
-		batchFullCh:  make(chan struct{}, 1),
+		stopCh:      make(chan context.Context, 1),
-		stopCh:       make(chan context.Context, 1),
+		done:        make(chan struct{}),
-		done:         make(chan struct{}),
 	}
-	h.batchTaken.L = &h.mu
 
 	// Start the flush timer goroutine
 	go flushLoop(batchTimeout, h.batchFullCh, h.stopCh, h.done, h.flushBatch)
@@ -115,48 +99,25 @@ func (h *ASHandler) QueueCapacity() int {
 	return asHandlerQueueSize
 }
 
-// HandleMessage processes a RIS Live BGP message by extracting all ASNs from
+// HandleMessage adds the time of msg to the batch entry of each AS in its AS
-// the AS path and queuing them for batch insertion into the database. The
+// path, the origin AS and the members of an AS set included. When the batch
-// origin ASN and all transit ASNs are recorded with their associated
+// holds asnBatchSize ASes, it wakes the flush goroutine to write it.
-// timestamps. When the batch reaches the configured size threshold, the flush
-// goroutine is woken to write it.
 func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
-	// Use the pre-parsed timestamp
-	timestamp := msg.ParsedTimestamp
-
-	originASN := msg.Path.Origin()
-
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
-	// While the batch is at its maximum size, wait until the flush goroutine
+	// The origin AS is the last AS of the sequence, so the loop adds it too.
-	// takes it. Meanwhile the handler takes nothing off its queue, which
-	// fills, and the streamer drops messages for it.
-	for len(h.batch) >= h.batchMaxSize {
-		h.batchTaken.Wait()
-	}
-
-	// Queue origin ASN operation
-	if originASN > 0 {
-		h.batch = append(h.batch, asnOp{
-			number:    originASN,
-			timestamp: timestamp,
-		})
-	}
-
-	// Also track all ASNs in the path, the members of an AS set included
 	for _, asn := range slices.Concat(msg.Path.Sequence, msg.Path.Set) {
 		if asn > 0 {
-			h.batch = append(h.batch, asnOp{
+			seen := h.batch[asn]
-				number:    asn,
+			seen.Add(msg.ParsedTimestamp)
-				timestamp: timestamp,
+			h.batch[asn] = seen
-			})
 		}
 	}
 
 	// A full batch wakes the flush goroutine. If it is writing, the wake-up
-	// waits in batchFullCh and the batch keeps growing until the write ends
+	// waits in batchFullCh and the batch keeps taking updates until the write
-	// or the batch reaches batchMaxSize.
+	// ends.
 	if len(h.batch) >= asnBatchSize {
 		select {
 		case h.batchFullCh <- struct{}{}:
@@ -165,10 +126,10 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
 	}
 }
 
-// Stop stops the flush goroutine, which writes the operations the handler holds
+// Stop stops the flush goroutine, which writes the ASes the handler holds before
-// before it returns, and waits for it until ctx ends. Once ctx has ended the
+// it returns, and waits for it until ctx ends. Once ctx has ended the flush
-// flush goroutine skips that last write, and a write still running is lost
+// goroutine skips that last write, and a write still running is lost when the
-// when the process exits, so Stop logs how many operations are not written.
+// process exits, so Stop logs how many ASes are not written.
 func (h *ASHandler) Stop(ctx context.Context) {
 	h.stopCh <- ctx
 
@@ -183,14 +144,14 @@ func (h *ASHandler) Stop(ctx context.Context) {
 
 	if unwritten > 0 {
 		h.logger.Error("Stop deadline passed before the AS handler wrote "+
-			"its updates", "updates", unwritten)
+			"its ASes", "asns", unwritten)
 	}
 }
 
-// takeBatch returns the pending operations and leaves an empty batch in their
+// takeBatch returns the pending ASes and leaves an empty batch in their place
-// place for HandleMessage to fill. They count in writing until flushBatch has
+// for HandleMessage to fill. They count in writing until flushBatch has written
-// written them.
+// them.
-func (h *ASHandler) takeBatch() []asnOp {
+func (h *ASHandler) takeBatch() map[int]database.SeenTimes {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
@@ -200,8 +161,7 @@ func (h *ASHandler) takeBatch() []asnOp {
 
 	batch := h.batch
 	h.writing = len(batch)
-	h.batch = make([]asnOp, 0, asnBatchSize)
+	h.batch = make(map[int]database.SeenTimes)
-	h.batchTaken.Broadcast()
 
 	return batch
 }
@@ -214,21 +174,11 @@ func (h *ASHandler) flushBatch() {
 		return
 	}
 
-	// The earliest and the latest time the batch saw each AS. The batch holds
-	// several updates for an AS, from many peers, so a new AS is stored first
-	// seen at the earliest of them and last seen at the latest.
-	asns := make(map[int]database.SeenTimes)
-	for _, op := range batch {
-		seen := asns[op.number]
-		seen.Add(op.timestamp)
-		asns[op.number] = seen
-	}
-
 	// Process all ASNs in a single batch transaction
-	err := h.db.GetOrCreateASNBatch(asns)
+	err := h.db.GetOrCreateASNBatch(batch)
 	if err != nil {
 		h.logger.Error("Failed to process ASN batch",
-			"error", err, "count", len(asns))
+			"error", err, "count", len(batch))
 	}
 
 	h.mu.Lock()

internal/routewatch/ashandler_internal_test.go

@@ -5,28 +5,43 @@ import (
 	"slices"
 	"testing"
 
+	"git.eeqj.de/sneak/routewatch/internal/database"
 	"git.eeqj.de/sneak/routewatch/internal/logger"
+	"git.eeqj.de/sneak/routewatch/internal/ristypes"
 )
 
+// pathOfASes returns an AS path of n different ASes: AS 1 to AS n. With n up to
+// asnBatchSize, none of them is testASNA to testASND.
+func pathOfASes(n int) ristypes.ASPath {
+	sequence := make([]int, n)
+	for i := range sequence {
+		sequence[i] = i + 1
+	}
+
+	return ristypes.ASPath{Sequence: sequence}
+}
+
 // TestASHandlerStoresTheEarliestAndLatestTimesOfANewAS handles two updates
-// whose path carries an AS the database does not hold yet, the second 5
+// whose path carries an AS the database does not hold yet, in one batch into a
-// seconds after the first, in one batch into a real database. The AS is stored
+// real database. The second is 5 seconds after the first, and the handler gets
-// first seen at the time of the first update and last seen at the time of the
+// it first. The AS is stored first seen at the time of the first update and
-// second.
+// last seen at the time of the second.
 func TestASHandlerStoresTheEarliestAndLatestTimesOfANewAS(t *testing.T) {
 	t.Parallel()
 
 	db := newTestDatabase(t)
 
 	// Built without NewASHandler's flush goroutine, so the test flushes itself.
-	h := &ASHandler{db: db, logger: logger.New(), batchMaxSize: asnBatchMaxSize}
+	h := &ASHandler{
+		db: db, logger: logger.New(), batch: make(map[int]database.SeenTimes),
+	}
 
 	first := peerAnnouncement(testPeerIP, testV4Prefix)
 	second := peerAnnouncement(testPeerIP, testV4Prefix)
 	second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
 
-	h.HandleMessage(first)
 	h.HandleMessage(second)
+	h.HandleMessage(first)
 	h.flushBatch()
 
 	asn, _, err := db.GetAS(t.Context(), testASNA)
@@ -54,7 +69,9 @@ func TestASHandlerRecordsNothingFromAPathWithASZero(t *testing.T) {
 	store := newMockStore()
 
 	// Built without NewASHandler's flush goroutine, so the test flushes itself.
-	h := &ASHandler{db: store, logger: logger.New(), batchMaxSize: asnBatchMaxSize}
+	h := &ASHandler{
+		db: store, logger: logger.New(), batch: make(map[int]database.SeenTimes),
+	}
 
 	msg := peerAnnouncement(testPeerIP, testV4Prefix)
 	msg.Path = decodePath(t, testASNA, 0, testASNB)

internal/routewatch/batchwrite_internal_test.go

@@ -178,6 +178,7 @@ type stopLog struct {
 	Updates      int    `json:"updates"`
 	TableChanges int    `json:"table_changes"`
 	Paths        int    `json:"paths"`
+	ASNs         int    `json:"asns"`
 }
 
 // newErrorLogger returns a logger that writes only errors, as JSON lines, to
@@ -495,20 +496,20 @@ func TestPrefixHandlerStopCountsTheTableChangesOfAWriteInProgress(t *testing.T)
 	}
 }
 
+// TestASHandlerSkipsItsLastWriteAfterTheStopDeadline also checks what Stop
+// logs: the asnBatchSize ASes of the held write and testASNB, handled since,
+// are not written.
 func TestASHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
 	t.Parallel()
 
-	// Each message gives an update for its origin AS and one for each AS
+	const wantASNs = asnBatchSize + 1
-	// in its path: fill asnBatchSize+1, extra 2.
-	const wantUpdates = asnBatchSize + 1 + 2
 
 	var logs bytes.Buffer
 
 	store := newBlockingStore()
 	h := newASHandler(store, newErrorLogger(&logs), longerThanAnyTest)
-	fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
 	stopWhileAWriteIsBlocked(t, store, h, h.done,
-		[]*ristypes.RISMessage{{Path: fill}},
+		[]*ristypes.RISMessage{{Path: pathOfASes(asnBatchSize)}},
 		&ristypes.RISMessage{Path: ristypes.ASPath{Sequence: []int{testASNB}}})
 
 	store.mu.Lock()
@@ -518,26 +519,27 @@ func TestASHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
 		t.Errorf("AS%d was written after the stop deadline", testASNB)
 	}
 
-	if logged := lastLogged(t, &logs); logged.Updates != wantUpdates {
+	if logged := lastLogged(t, &logs); logged.ASNs != wantASNs {
-		t.Errorf("Stop logged %d updates not written, want %d",
+		t.Errorf("Stop logged %d ASes not written, want %d",
-			logged.Updates, wantUpdates)
+			logged.ASNs, wantASNs)
 	}
 }
 
 // TestASHandlerSkipsAFullBatchAfterTheStopDeadline hands the AS handler a full
 // batch while a write is held, so the batch's wake-up of the flush goroutine
-// waits. The stop deadline passes meanwhile. When the write ends, the flush
+// waits. The full batch holds the fill's ASes again, and testASNB. The stop
-// goroutine must stop instead of writing the batch.
+// deadline passes meanwhile. When the write ends, the flush goroutine must stop
+// instead of writing the batch.
 func TestASHandlerSkipsAFullBatchAfterTheStopDeadline(t *testing.T) {
 	t.Parallel()
 
 	store := newBlockingStore()
 	h := newASHandler(store, logger.New(), longerThanAnyTest)
-	fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
+	fill := &ristypes.RISMessage{Path: pathOfASes(asnBatchSize)}
-	full := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNB}, asnBatchSize)}
 	stopWhileAWriteIsBlocked(t, store, h, h.done,
-		[]*ristypes.RISMessage{{Path: fill}},
+		[]*ristypes.RISMessage{fill},
-		&ristypes.RISMessage{Path: full})
+		fill,
+		&ristypes.RISMessage{Path: ristypes.ASPath{Sequence: []int{testASNB}}})
 
 	store.mu.Lock()
 	defer store.mu.Unlock()
@@ -703,9 +705,8 @@ func TestASHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) {
 
 	store := newBlockingStore()
 	h := newASHandler(store, logger.New(), longerThanAnyTest)
-	fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
 	handleWhileAWriteIsBlocked(t, store, h,
-		[]*ristypes.RISMessage{{Path: fill}},
+		[]*ristypes.RISMessage{{Path: pathOfASes(asnBatchSize)}},
 		&ristypes.RISMessage{Path: ristypes.ASPath{Sequence: []int{testASNB}}})
 
 	store.mu.Lock()
@@ -762,25 +763,41 @@ func TestPrefixHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
 	}
 }
 
-func TestASHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
+// TestASHandlerNeverWaitsForAWrite starts a write that the store holds. Then it
+// hands the AS handler twenty messages whose paths each hold one AS
+// asnBatchSize times. HandleMessage must return each time, and the batch must
+// hold one entry, for that AS.
+func TestASHandlerNeverWaitsForAWrite(t *testing.T) {
 	t.Parallel()
 
+	const messages = 20
+
 	store := newBlockingStore()
 	h := newASHandler(store, logger.New(), longerThanAnyTest)
-	h.batchMaxSize = asnBatchSize
-	fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
-	toMax := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
-	handleAtMaxBatchSize(t, store, h,
-		[]*ristypes.RISMessage{{Path: fill}},
-		[]*ristypes.RISMessage{{Path: toMax}},
-		&ristypes.RISMessage{Path: ristypes.ASPath{Sequence: []int{testASNB}}})
 
-	store.mu.Lock()
+	defer stopInTime(t, h)
-	defer store.mu.Unlock()
+	defer store.releaseWrites()
 
-	if !store.asns[testASNB] {
+	startBlockedWrite(t, store, h,
-		t.Errorf("Stop did not write AS%d, handled after the batch reached its "+
+		[]*ristypes.RISMessage{{Path: pathOfASes(asnBatchSize)}})
-			"maximum size", testASNB)
+
+	oneAS := &ristypes.RISMessage{
+		Path: ristypes.ASPath{Sequence: slices.Repeat([]int{testASNB}, asnBatchSize)},
+	}
+
+	if !returnsInTime(func() {
+		for range messages {
+			h.HandleMessage(oneAS)
+		}
+	}) {
+		t.Fatal("HandleMessage did not return while a database write was blocked")
+	}
+
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	if len(h.batch) != 1 {
+		t.Errorf("the batch holds %d entries for one AS, want 1", len(h.batch))
 	}
 }
 
@@ -844,13 +861,11 @@ func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) {
 			batchSize: asnBatchSize,
 			handle: func(n int) chan struct{} {
 				h := &ASHandler{
-					batchMaxSize: asnBatchMaxSize,
+					batch:       make(map[int]database.SeenTimes),
-					batchFullCh:  make(chan struct{}, 1),
+					batchFullCh: make(chan struct{}, 1),
 				}
-				// The origin AS and each AS in the path are one update each.
+				// Each of the path's n ASes is one entry of the batch.
-				h.HandleMessage(&ristypes.RISMessage{
+				h.HandleMessage(&ristypes.RISMessage{Path: pathOfASes(n)})
-					Path: ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, n-1)},
-				})
 
 				return h.batchFullCh
 			},