Bound each shutdown stage by the stop deadline (closes #100)
check / check (push) Canceled after 0s

Shutdown passes its context to every stage: the streamer, the four
handlers, the WHOIS fetcher, the database maintainer and the HTTP server
each stop waiting when it ends. A handler then skips its last write and
logs how many updates it did not write, and the HTTP server closes the
connections still open. The database is checkpointed and closed last,
and left open once the deadline has passed, since a write may still
hold it. The stop hook gives Shutdown 8 seconds and fx 9, inside the 10
seconds upaas and docker stop allow, so a stop signal exits 0.

Model: opus-5-5
This commit was merged in pull request #130.
This commit is contained in:
2026-10-20 02:15:10 +02:00
parent 237f73fc49
commit 09d3073fa5
20 changed files with 1097 additions and 130 deletions
+46 -20
View File
@@ -1,6 +1,7 @@
package routewatch
import (
"context"
"slices"
"sync"
"time"
@@ -39,18 +40,23 @@ type ASHandler struct {
db database.Store
logger *logger.Logger
// Batching. mu guards batch only: the flush goroutine takes the batch
// under mu and writes it after releasing mu, so HandleMessage waits for
// a database write only while batch is at asnBatchMaxSize.
// Batching. mu guards batch and writing only: the flush goroutine takes
// the batch under mu and writes it after releasing mu, so HandleMessage
// waits for a database write only while batch is at asnBatchMaxSize.
mu sync.Mutex
batch []asnOp
// writing is how many operations the flush goroutine took from batch
// and has not finished writing.
writing int
// batchTaken is signalled when the flush goroutine takes batch; its L is
// &mu.
batchTaken sync.Cond
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
lastFlush time.Time // used only by the flush goroutine
stopCh chan struct{}
wg sync.WaitGroup
// stopCh hands Stop's context to the flush goroutine. It holds one, so
// Stop does not wait for a write in progress to hand it over.
stopCh chan context.Context
done chan struct{} // closed when the flush goroutine returns
}
type asnOp struct {
@@ -70,12 +76,12 @@ func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
batch: make([]asnOp, 0, asnBatchSize),
batchFullCh: make(chan struct{}, 1),
lastFlush: time.Now(),
stopCh: make(chan struct{}),
stopCh: make(chan context.Context, 1),
done: make(chan struct{}),
}
h.batchTaken.L = &h.mu
// Start the flush timer goroutine
h.wg.Add(1)
go h.flushLoop()
return h
@@ -147,20 +153,32 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
}
}
// Stop gracefully shuts down the ASHandler by signaling the background flush
// goroutine to terminate and waiting for it to complete. Any pending ASN
// operations in the current batch are flushed to the database before Stop
// returns. This method should be called during application shutdown to ensure
// no data is lost.
func (h *ASHandler) Stop() {
close(h.stopCh)
h.wg.Wait()
// Stop stops the flush goroutine, which writes the operations the handler holds
// before it returns, and waits for it until ctx ends. Once ctx has ended the
// flush goroutine skips that last write, and a write still running is lost
// when the process exits, so Stop logs how many operations are not written.
func (h *ASHandler) Stop(ctx context.Context) {
h.stopCh <- ctx
select {
case <-h.done:
case <-ctx.Done():
}
h.mu.Lock()
unwritten := h.writing + len(h.batch)
h.mu.Unlock()
if unwritten > 0 {
h.logger.Error("Stop deadline passed before the AS handler wrote "+
"its updates", "updates", unwritten)
}
}
// flushLoop runs in a goroutine, the only one that writes the batch: it
// flushes when the batch is full, periodically, and once more on Stop.
func (h *ASHandler) flushLoop() {
defer h.wg.Done()
defer close(h.done)
ticker := time.NewTicker(asnBatchTimeout)
defer ticker.Stop()
@@ -173,9 +191,11 @@ func (h *ASHandler) flushLoop() {
}
case <-h.batchFullCh:
h.flushBatch()
case <-h.stopCh:
// Final flush
h.flushBatch()
case ctx := <-h.stopCh:
// Final flush, unless the stop deadline has passed
if ctx.Err() == nil {
h.flushBatch()
}
return
}
@@ -183,7 +203,8 @@ func (h *ASHandler) flushLoop() {
}
// takeBatch returns the pending operations and leaves an empty batch in their
// place for HandleMessage to fill.
// place for HandleMessage to fill. They count in writing until flushBatch has
// written them.
func (h *ASHandler) takeBatch() []asnOp {
h.mu.Lock()
defer h.mu.Unlock()
@@ -193,6 +214,7 @@ func (h *ASHandler) takeBatch() []asnOp {
}
batch := h.batch
h.writing = len(batch)
h.batch = make([]asnOp, 0, asnBatchSize)
h.batchTaken.Broadcast()
@@ -222,5 +244,9 @@ func (h *ASHandler) flushBatch() {
"error", err, "count", len(asnMap))
}
h.mu.Lock()
h.writing = 0
h.mu.Unlock()
h.lastFlush = time.Now()
}