2 Commits
Author SHA1 Message Date
sneak 29d3f30d23 Re-vendor the shared files from sneak/prompts at dd4027b (closes #52)
check / check (push) Canceled after 0s
Takes .golangci.yml, .dockerignore, .gitignore, .editorconfig, the
workflow and REPO_POLICIES.md from sneak/prompts dd4027b, keeping this
repository's own entries after the shared content. Lint (golangci-lint
v2.14.0) and test become Dockerfile phases that script/lint and
script/test build alone and uncached; the image build depends on both,
and script/cibuild bootstraps, runs script/check, then builds the image.
The agent guidance file moves to AGENTS.md. The Go code is fixed for
what the new configuration finds; reads that iterate rows now also
report an iteration error.

Model: opus-5-5
2026-10-14 19:11:06 +00:00
clawbot e0b9c2685f Reconnect when RIS Live ends or stalls the stream (closes #53)
check / check (push) Canceled after 0s
When RIS Live ended the response normally, the reconnect loop stopped for good and the health check kept reporting the feed connected; a connection that stayed open but sent nothing blocked forever. Now only `Stop` ends the loop: any other end clears the connected flag and reconnects with the existing backoff. If no line arrives for 60 seconds, including while waiting for RIS Live to answer the request, the streamer drops the connection and reconnects. Tests use a local fake server. README "Reconnection" describes both.

Model: opus-5-5
2026-10-14 19:42:57 +02:00
3 changed files with 224 additions and 29 deletions

No files matched your search

+4 -1
View File
@@ -141,7 +141,10 @@ routewatch/
- **Batched Writes**: All database operations are batched for performance - **Batched Writes**: All database operations are batched for performance
- **Backpressure**: Probabilistic message dropping when queues exceed 50% capacity - **Backpressure**: Probabilistic message dropping when queues exceed 50% capacity
- **Graceful Shutdown**: 60-second timeout, flushes all pending batches - **Graceful Shutdown**: 60-second timeout, flushes all pending batches
- **Reconnection**: Exponential backoff (5s-320s) with reset after 30s of stable connection - **Reconnection**: Any end of the stream other than shutdown, including RIS Live
ending the response normally, marks the feed disconnected and reconnects with
exponential backoff (5s-320s). The backoff resets after a stream that lasted
more than 30s. A stream with no line for 60s is dropped and reconnected.
- **IP Lookup**: the most specific live route is found by looking up the - **IP Lookup**: the most specific live route is found by looking up the
address's prefix at each mask length, longest first, in the prefix index address's prefix at each mask length, longest first, in the prefix index
(at most 25 lookups for IPv4, 113 for IPv6); prefixes are stored in the (at most 25 lookups for IPv4, 113 for IPv6); prefixes are stored in the
+44 -16
View File
@@ -50,6 +50,7 @@ const (
metricsUpdateRate = time.Second metricsUpdateRate = time.Second
minBackoffDelay = 5 * time.Second minBackoffDelay = 5 * time.Second
maxBackoffDelay = 320 * time.Second maxBackoffDelay = 320 * time.Second
streamIdleTimeout = 60 * time.Second // Reconnect when no line arrives for this long
metricsLogInterval = 10 * time.Second metricsLogInterval = 10 * time.Second
bytesPerKB = 1024 bytesPerKB = 1024
bytesPerMB = 1024 * 1024 bytesPerMB = 1024 * 1024
@@ -63,6 +64,11 @@ const (
var ( var (
errAlreadyRunning = errors.New("streamer already running") errAlreadyRunning = errors.New("streamer already running")
errUnexpectedStatus = errors.New("unexpected status code") errUnexpectedStatus = errors.New("unexpected status code")
// errStreamEnded is returned by stream when RIS Live ends the response.
errStreamEnded = errors.New("RIS Live ended the stream")
// errStreamIdle is returned by stream when no line arrived for the idle
// timeout.
errStreamIdle = errors.New("no line received")
) )
// MessageHandler defines the interface for processing RIS messages. // MessageHandler defines the interface for processing RIS messages.
@@ -115,6 +121,10 @@ type Streamer struct {
logger *logger.Logger logger *logger.Logger
client *http.Client client *http.Client
url string url string
// Drop the connection when no line arrives for this long
idleTimeout time.Duration
// First wait before reconnecting; doubles up to maxBackoffDelay
minBackoff time.Duration
handlers []*handlerInfo handlers []*handlerInfo
rawHandler RawMessageHandler rawHandler RawMessageHandler
mu sync.RWMutex mu sync.RWMutex
@@ -135,6 +145,8 @@ func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer {
return &Streamer{ return &Streamer{
logger: logger, logger: logger,
url: risLiveURL, url: risLiveURL,
idleTimeout: streamIdleTimeout,
minBackoff: minBackoffDelay,
client: &http.Client{ client: &http.Client{
Timeout: 0, // No timeout for streaming Timeout: 0, // No timeout for streaming
Transport: &http.Transport{ Transport: &http.Transport{
@@ -430,7 +442,7 @@ func (s *Streamer) updateMetrics(messageBytes int) {
// streamWithReconnect handles streaming with automatic reconnection and // streamWithReconnect handles streaming with automatic reconnection and
// exponential backoff // exponential backoff
func (s *Streamer) streamWithReconnect(ctx context.Context) { func (s *Streamer) streamWithReconnect(ctx context.Context) {
backoffDelay := minBackoffDelay backoffDelay := s.minBackoff
consecutiveFailures := 0 consecutiveFailures := 0
for { for {
@@ -447,8 +459,9 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) {
err := s.stream(ctx) err := s.stream(ctx)
streamDuration := time.Since(startTime) streamDuration := time.Since(startTime)
if err == nil { // Only Stop ends the streamer. A stream that ends any other way,
// Clean exit (context cancelled) // with or without an error, is a disconnect and is reconnected.
if ctx.Err() != nil {
return return
} }
@@ -459,18 +472,14 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) {
"stream_duration", streamDuration) "stream_duration", streamDuration)
s.metrics.SetConnected(false) s.metrics.SetConnected(false)
// Check if context is cancelled // A stream that lasted more than 30 seconds resets the backoff. That
if ctx.Err() != nil { // includes a stream dropped after streamIdleTimeout with no line, so
return // a silent feed is retried after the first backoff delay every time.
}
// If we streamed for more than 30 seconds, reset the backoff
// This indicates we had a successful connection that received data
if streamDuration > 30*time.Second { if streamDuration > 30*time.Second {
s.logger.Info("Resetting backoff delay due to successful connection", s.logger.Info("Resetting backoff delay due to successful connection",
"stream_duration", streamDuration) "stream_duration", streamDuration)
backoffDelay = minBackoffDelay backoffDelay = s.minBackoff
consecutiveFailures = 0 consecutiveFailures = 0
} else { } else {
// Increment consecutive failures // Increment consecutive failures
@@ -503,7 +512,13 @@ func (s *Streamer) stream(ctx context.Context) error {
connCtx, connCancel := context.WithCancel(ctx) connCtx, connCancel := context.WithCancel(ctx)
defer connCancel() defer connCancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil) // The idle timer drops the connection, by cancelling connCtx, when no
// line arrives for s.idleTimeout. Each line read starts it over. It
// also covers the wait for the server to answer.
idleTimer := time.AfterFunc(s.idleTimeout, connCancel)
defer idleTimer.Stop()
req, err := http.NewRequestWithContext(connCtx, http.MethodGet, s.url, nil)
if err != nil { if err != nil {
return fmt.Errorf("failed to create request: %w", err) return fmt.Errorf("failed to create request: %w", err)
} }
@@ -565,7 +580,7 @@ func (s *Streamer) stream(ctx context.Context) error {
// Wire byte update ticker - update metrics with actual wire bytes periodically // Wire byte update ticker - update metrics with actual wire bytes periodically
go s.recordWireBytes(connCtx, wireCounter) go s.recordWireBytes(connCtx, wireCounter)
return s.readStream(ctx, reader) return s.readStream(ctx, connCtx, reader, idleTimer)
} }
// recordWireBytes adds the bytes wireCounter has counted to the metrics once a // recordWireBytes adds the bytes wireCounter has counted to the metrics once a
@@ -596,11 +611,16 @@ func (s *Streamer) recordWireBytes(
} }
// readStream hands each line of the stream to processLine until the stream // readStream hands each line of the stream to processLine until the stream
// ends, a line fails, or ctx is cancelled. // ends, a line fails, or ctx is cancelled. Each line restarts idleTimer, which
func (s *Streamer) readStream(ctx context.Context, reader io.Reader) error { // cancels connCtx when it fires.
func (s *Streamer) readStream(
ctx, connCtx context.Context, reader io.Reader, idleTimer *time.Timer,
) error {
scanner := bufio.NewScanner(reader) scanner := bufio.NewScanner(reader)
for scanner.Scan() { for scanner.Scan() {
idleTimer.Reset(s.idleTimeout)
select { select {
case <-ctx.Done(): case <-ctx.Done():
s.logger.Info("Stream stopped by context") s.logger.Info("Stream stopped by context")
@@ -620,12 +640,20 @@ func (s *Streamer) readStream(ctx context.Context, reader io.Reader) error {
} }
} }
if ctx.Err() != nil {
return ctx.Err()
}
if connCtx.Err() != nil {
return fmt.Errorf("%w for %s", errStreamIdle, s.idleTimeout)
}
err := scanner.Err() err := scanner.Err()
if err != nil { if err != nil {
return fmt.Errorf("scanner error: %w", err) return fmt.Errorf("scanner error: %w", err)
} }
return nil return errStreamEnded
} }
// processLine passes one line of the stream to the raw handler, parses it, // processLine passes one line of the stream to the raw handler, parses it,
+176 -12
View File
@@ -7,6 +7,7 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"runtime" "runtime"
"sync/atomic"
"testing" "testing"
"time" "time"
@@ -41,6 +42,14 @@ func TestNewStreamer(t *testing.T) {
if s.metrics != metricsTracker { if s.metrics != metricsTracker {
t.Error("metrics tracker not set correctly") t.Error("metrics tracker not set correctly")
} }
if s.idleTimeout != streamIdleTimeout {
t.Errorf("idleTimeout = %s, want %s", s.idleTimeout, streamIdleTimeout)
}
if s.minBackoff != minBackoffDelay {
t.Errorf("minBackoff = %s, want %s", s.minBackoff, minBackoffDelay)
}
} }
// TestStreamDoesNotLeakTickersAcrossReconnects drives many short-lived // TestStreamDoesNotLeakTickersAcrossReconnects drives many short-lived
@@ -63,8 +72,8 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
// One warm-up connection so any persistent HTTP transport goroutine exists // One warm-up connection so any persistent HTTP transport goroutine exists
// before we take the baseline. // before we take the baseline.
err := s.stream(context.Background()) err := s.stream(context.Background())
if err != nil { if !errors.Is(err, errStreamEnded) {
t.Fatalf("warm-up stream returned error: %v", err) t.Fatalf("warm-up stream returned %v, want %v", err, errStreamEnded)
} }
s.client.CloseIdleConnections() s.client.CloseIdleConnections()
@@ -74,8 +83,8 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
const reconnects = 20 const reconnects = 20
for range reconnects { for range reconnects {
err := s.stream(context.Background()) err := s.stream(context.Background())
if err != nil { if !errors.Is(err, errStreamEnded) {
t.Fatalf("stream returned error: %v", err) t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
} }
} }
@@ -87,17 +96,39 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
} }
} }
// updateHandler wants UPDATE messages and does nothing with them. // updateLine is one UPDATE message as RIS Live sends it.
type updateHandler struct{} const updateLine = `{"type":"ris_message","data":{"type":"UPDATE",` +
`"peer":"192.0.2.1","peer_asn":"64496","timestamp":1700000000}}` + "\n"
// updateHandler wants UPDATE messages. For each one it handles it sends on
// handled if there is room, and otherwise does nothing; a nil handled is
// never sent on.
type updateHandler struct {
handled chan struct{}
}
func (updateHandler) WantsMessage(messageType string) bool { func (updateHandler) WantsMessage(messageType string) bool {
return messageType == "UPDATE" return messageType == "UPDATE"
} }
func (updateHandler) HandleMessage(*ristypes.RISMessage) {}
func (updateHandler) QueueCapacity() int { return 10 } func (updateHandler) QueueCapacity() int { return 10 }
func (h updateHandler) HandleMessage(*ristypes.RISMessage) {
select {
case h.handled <- struct{}{}:
default:
}
}
// flush sends what w has buffered to the client. The writers httptest servers
// hand to handlers can always do that.
func flush(w http.ResponseWriter) {
flusher, ok := w.(http.Flusher)
if ok {
flusher.Flush()
}
}
// TestStopBeforeMessageReachesQueues stops the streamer after the read loop // TestStopBeforeMessageReachesQueues stops the streamer after the read loop
// has checked for cancellation but before it hands the message to the handler // has checked for cancellation but before it hands the message to the handler
// queues. That is the gap a stop from another goroutine can land in, and it // queues. That is the gap a stop from another goroutine can land in, and it
@@ -106,12 +137,9 @@ func (updateHandler) QueueCapacity() int { return 10 }
func TestStopBeforeMessageReachesQueues(t *testing.T) { func TestStopBeforeMessageReachesQueues(t *testing.T) {
t.Parallel() t.Parallel()
const line = `{"type":"ris_message","data":{"type":"UPDATE",` +
`"peer":"192.0.2.1","peer_asn":"64496","timestamp":1700000000}}` + "\n"
srv := httptest.NewServer(http.HandlerFunc( srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) { func(w http.ResponseWriter, _ *http.Request) {
_, _ = io.WriteString(w, line) _, _ = io.WriteString(w, updateLine)
})) }))
defer srv.Close() defer srv.Close()
@@ -135,6 +163,142 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) {
s.Stop() s.Stop()
} }
// TestReconnectWhenStreamEnds has the server end the first response
// normally, as when RIS Live restarts. The streamer must treat that as a
// disconnect and connect again.
func TestReconnectWhenStreamEnds(t *testing.T) {
t.Parallel()
endResponse := func(http.ResponseWriter, *http.Request) {}
checkReconnect(t, endResponse, streamIdleTimeout)
}
// TestReconnectWhenStreamGoesIdle has the server hold the first response
// open without sending anything. The streamer must drop it once the idle
// timeout passes and connect again.
func TestReconnectWhenStreamGoesIdle(t *testing.T) {
t.Parallel()
const idleTimeout = 50 * time.Millisecond
holdResponse := func(w http.ResponseWriter, r *http.Request) {
flush(w)
<-r.Context().Done()
}
checkReconnect(t, holdResponse, idleTimeout)
}
// TestReconnectWhenServerNeverAnswers has the server accept the first
// request and never answer it, not even with headers. The streamer must give
// up on it once the idle timeout passes and connect again.
func TestReconnectWhenServerNeverAnswers(t *testing.T) {
t.Parallel()
const idleTimeout = 50 * time.Millisecond
neverAnswer := func(_ http.ResponseWriter, r *http.Request) {
<-r.Context().Done()
}
checkReconnect(t, neverAnswer, idleTimeout)
}
// TestStreamNotDroppedWhileLinesArrive has the server send lines well within
// the idle timeout for several times the timeout, then end the response. Each
// line must start the idle timeout over, so the stream lasts until the server
// ends it instead of being dropped as idle.
func TestStreamNotDroppedWhileLinesArrive(t *testing.T) {
t.Parallel()
const (
idleTimeout = 100 * time.Millisecond
lineInterval = idleTimeout / 10
sendFor = 5 * idleTimeout
)
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
deadline := time.Now().Add(sendFor)
for time.Now().Before(deadline) {
_, _ = io.WriteString(w, updateLine)
flush(w)
time.Sleep(lineInterval)
}
}))
defer srv.Close()
s := New(logger.New(), metrics.New())
s.url = srv.URL
s.idleTimeout = idleTimeout
err := s.stream(context.Background())
if !errors.Is(err, errStreamEnded) {
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
}
}
// checkReconnect runs a streamer against a fake RIS Live server. The first
// response is served by firstResponse; every later one carries one UPDATE
// line and then stays open. The streamer must come back for a second
// response, must not be marked connected when it does, and must handle the
// UPDATE in it.
func checkReconnect(
t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration,
) {
t.Helper()
const maxWait = 5 * time.Second
tracker := metrics.New()
connectedAtReconnect := make(chan bool, 1)
var requests atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if requests.Add(1) == 1 {
firstResponse(w, r)
return
}
select {
case connectedAtReconnect <- tracker.IsConnected():
default:
}
_, _ = io.WriteString(w, updateLine)
flush(w)
<-r.Context().Done()
}))
defer srv.Close()
s := New(logger.New(), tracker)
s.url = srv.URL
s.idleTimeout = idleTimeout
s.minBackoff = time.Millisecond
handler := updateHandler{handled: make(chan struct{}, 1)}
s.RegisterHandler(handler)
err := s.Start(t.Context())
if err != nil {
t.Fatalf("Start returned error: %v", err)
}
defer s.Stop()
select {
case <-handler.handled:
case <-time.After(maxWait):
t.Fatalf("no message from a second response handled within %s", maxWait)
}
if <-connectedAtReconnect {
t.Error("streamer was still marked connected when it came back " +
"for a second response")
}
}
// settledGoroutineCount lets transient goroutines finish, then reports the // settledGoroutineCount lets transient goroutines finish, then reports the
// current count. // current count.
func settledGoroutineCount() int { func settledGoroutineCount() int {