Bound each shutdown stage by the stop deadline (closes #100)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
Shutdown passes its context to every stage: the streamer, the four handlers, the WHOIS fetcher, the database maintainer and the HTTP server each stop waiting when it ends. A handler then skips its last write and logs how many updates it did not write, and the HTTP server closes the connections still open. The database is checkpointed and closed last, and left open once the deadline has passed, since a write may still hold it. The stop hook gives Shutdown 8 seconds and fx 9, inside the 10 seconds upaas and docker stop allow, so a stop signal exits 0. Model: opus-5-5
This commit was merged in pull request #130.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package routewatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -39,18 +40,23 @@ type ASHandler struct {
|
||||
db database.Store
|
||||
logger *logger.Logger
|
||||
|
||||
// Batching. mu guards batch only: the flush goroutine takes the batch
|
||||
// under mu and writes it after releasing mu, so HandleMessage waits for
|
||||
// a database write only while batch is at asnBatchMaxSize.
|
||||
// Batching. mu guards batch and writing only: the flush goroutine takes
|
||||
// the batch under mu and writes it after releasing mu, so HandleMessage
|
||||
// waits for a database write only while batch is at asnBatchMaxSize.
|
||||
mu sync.Mutex
|
||||
batch []asnOp
|
||||
// writing is how many operations the flush goroutine took from batch
|
||||
// and has not finished writing.
|
||||
writing int
|
||||
// batchTaken is signalled when the flush goroutine takes batch; its L is
|
||||
// &mu.
|
||||
batchTaken sync.Cond
|
||||
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
|
||||
lastFlush time.Time // used only by the flush goroutine
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
// stopCh hands Stop's context to the flush goroutine. It holds one, so
|
||||
// Stop does not wait for a write in progress to hand it over.
|
||||
stopCh chan context.Context
|
||||
done chan struct{} // closed when the flush goroutine returns
|
||||
}
|
||||
|
||||
type asnOp struct {
|
||||
@@ -70,12 +76,12 @@ func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
|
||||
batch: make([]asnOp, 0, asnBatchSize),
|
||||
batchFullCh: make(chan struct{}, 1),
|
||||
lastFlush: time.Now(),
|
||||
stopCh: make(chan struct{}),
|
||||
stopCh: make(chan context.Context, 1),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
h.batchTaken.L = &h.mu
|
||||
|
||||
// Start the flush timer goroutine
|
||||
h.wg.Add(1)
|
||||
go h.flushLoop()
|
||||
|
||||
return h
|
||||
@@ -147,20 +153,32 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
|
||||
}
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the ASHandler by signaling the background flush
|
||||
// goroutine to terminate and waiting for it to complete. Any pending ASN
|
||||
// operations in the current batch are flushed to the database before Stop
|
||||
// returns. This method should be called during application shutdown to ensure
|
||||
// no data is lost.
|
||||
func (h *ASHandler) Stop() {
|
||||
close(h.stopCh)
|
||||
h.wg.Wait()
|
||||
// Stop stops the flush goroutine, which writes the operations the handler holds
|
||||
// before it returns, and waits for it until ctx ends. Once ctx has ended the
|
||||
// flush goroutine skips that last write, and a write still running is lost
|
||||
// when the process exits, so Stop logs how many operations are not written.
|
||||
func (h *ASHandler) Stop(ctx context.Context) {
|
||||
h.stopCh <- ctx
|
||||
|
||||
select {
|
||||
case <-h.done:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
|
||||
h.mu.Lock()
|
||||
unwritten := h.writing + len(h.batch)
|
||||
h.mu.Unlock()
|
||||
|
||||
if unwritten > 0 {
|
||||
h.logger.Error("Stop deadline passed before the AS handler wrote "+
|
||||
"its updates", "updates", unwritten)
|
||||
}
|
||||
}
|
||||
|
||||
// flushLoop runs in a goroutine, the only one that writes the batch: it
|
||||
// flushes when the batch is full, periodically, and once more on Stop.
|
||||
func (h *ASHandler) flushLoop() {
|
||||
defer h.wg.Done()
|
||||
defer close(h.done)
|
||||
|
||||
ticker := time.NewTicker(asnBatchTimeout)
|
||||
defer ticker.Stop()
|
||||
@@ -173,9 +191,11 @@ func (h *ASHandler) flushLoop() {
|
||||
}
|
||||
case <-h.batchFullCh:
|
||||
h.flushBatch()
|
||||
case <-h.stopCh:
|
||||
// Final flush
|
||||
h.flushBatch()
|
||||
case ctx := <-h.stopCh:
|
||||
// Final flush, unless the stop deadline has passed
|
||||
if ctx.Err() == nil {
|
||||
h.flushBatch()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -183,7 +203,8 @@ func (h *ASHandler) flushLoop() {
|
||||
}
|
||||
|
||||
// takeBatch returns the pending operations and leaves an empty batch in their
|
||||
// place for HandleMessage to fill.
|
||||
// place for HandleMessage to fill. They count in writing until flushBatch has
|
||||
// written them.
|
||||
func (h *ASHandler) takeBatch() []asnOp {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
@@ -193,6 +214,7 @@ func (h *ASHandler) takeBatch() []asnOp {
|
||||
}
|
||||
|
||||
batch := h.batch
|
||||
h.writing = len(batch)
|
||||
h.batch = make([]asnOp, 0, asnBatchSize)
|
||||
h.batchTaken.Broadcast()
|
||||
|
||||
@@ -222,5 +244,9 @@ func (h *ASHandler) flushBatch() {
|
||||
"error", err, "count", len(asnMap))
|
||||
}
|
||||
|
||||
h.mu.Lock()
|
||||
h.writing = 0
|
||||
h.mu.Unlock()
|
||||
|
||||
h.lastFlush = time.Now()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user