routewatch/internal/streamer/streamer_test.go

package streamer

import (
	"context"
	"errors"
	"io"
	"net/http"
	"net/http/httptest"
	"runtime"
	"sync/atomic"
	"testing"
	"time"

	"git.eeqj.de/sneak/routewatch/internal/logger"
	"git.eeqj.de/sneak/routewatch/internal/metrics"
	"git.eeqj.de/sneak/routewatch/internal/ristypes"
)

func TestNewStreamer(t *testing.T) {
	logger := logger.New()
	metricsTracker := metrics.New()
	s := New(logger, metricsTracker)

	if s == nil {
		t.Fatal("New() returned nil")
	}

	if s.logger != logger {
		t.Error("logger not set correctly")
	}

	if s.client == nil {
		t.Error("HTTP client not initialized")
	}

	if s.handlers == nil {
		t.Error("handlers slice not initialized")
	}

	if s.metrics != metricsTracker {
		t.Error("metrics tracker not set correctly")
	}

	if s.idleTimeout != streamIdleTimeout {
		t.Errorf("idleTimeout = %s, want %s", s.idleTimeout, streamIdleTimeout)
	}

	if s.minBackoff != minBackoffDelay {
		t.Errorf("minBackoff = %s, want %s", s.minBackoff, minBackoffDelay)
	}
}

// TestStreamDoesNotLeakTickersAcrossReconnects drives many short-lived
// connections (each stream call is one reconnect cycle) and asserts the
// goroutine count returns to its starting value. Each connection starts two
// ticker goroutines; before the fix they lived until the streamer's lifetime
// context was cancelled, so every reconnect leaked two.
func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
	// The handler returns immediately, so the response body is empty and each
	// stream call ends at once, standing in for a dropped connection.
	srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {}))
	defer srv.Close()

	s := New(logger.New(), metrics.New())
	s.url = srv.URL

	// One warm-up connection so any persistent HTTP transport goroutine exists
	// before we take the baseline.
	if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) {
		t.Fatalf("warm-up stream returned %v, want %v", err, errStreamEnded)
	}
	s.client.CloseIdleConnections()

	baseline := settledGoroutineCount()

	const reconnects = 20
	for range reconnects {
		if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) {
			t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
		}
	}
	s.client.CloseIdleConnections()

	if !waitForGoroutines(baseline) {
		t.Fatalf("goroutines did not return to baseline %d after %d reconnects, got %d",
			baseline, reconnects, runtime.NumGoroutine())
	}
}

// updateLine is one UPDATE message as RIS Live sends it.
const updateLine = `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` +
	`"peer_asn":"64496","timestamp":1700000000}}` + "\n"

// updateHandler wants UPDATE messages. For each one it handles it sends on
// handled if there is room, and otherwise does nothing; a nil handled is
// never sent on.
type updateHandler struct {
	handled chan struct{}
}

func (updateHandler) WantsMessage(messageType string) bool { return messageType == "UPDATE" }
func (updateHandler) QueueCapacity() int                   { return 10 }

func (h updateHandler) HandleMessage(*ristypes.RISMessage) {
	select {
	case h.handled <- struct{}{}:
	default:
	}
}

// stateDownLine is the STATE message RIS Live sends when the collector's BGP
// session with the peer stateDownPeer goes down.
const (
	stateDownPeer = "192.0.2.1"
	stateDownLine = `{"type":"ris_message","data":{"type":"STATE","peer":"` + stateDownPeer + `",` +
		`"peer_asn":"64496","timestamp":1700000000,"state":"down"}}` + "\n"
)

// stateHandler wants STATE messages only. Tests read what it is handed from
// its queue, so HandleMessage does nothing.
type stateHandler struct{}

func (stateHandler) WantsMessage(messageType string) bool {
	return messageType == ristypes.MessageTypeState
}
func (stateHandler) QueueCapacity() int                 { return 10 }
func (stateHandler) HandleMessage(*ristypes.RISMessage) {}

// TestStreamHandsStateMessagesToHandlersThatWantThem has the server send one
// STATE message whose state is down. The streamer must queue it, with its
// peer and state, for a handler that wants STATE messages.
func TestStreamHandsStateMessagesToHandlersThatWantThem(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		_, _ = io.WriteString(w, stateDownLine)
	}))
	defer srv.Close()

	s := New(logger.New(), metrics.New())
	s.url = srv.URL
	s.RegisterHandler(stateHandler{})

	if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) {
		t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
	}

	select {
	case msg := <-s.handlers[0].queue:
		if msg.Type != ristypes.MessageTypeState || msg.Peer != stateDownPeer || msg.State != ristypes.StateDown {
			t.Errorf("queued message has type %q, peer %q, state %q; want %q, %q, %q",
				msg.Type, msg.Peer, msg.State, ristypes.MessageTypeState, stateDownPeer, ristypes.StateDown)
		}
	default:
		t.Fatal("the STATE message was not queued for the handler that wants it")
	}
}

// streamWithFullStateQueue runs stream, in a goroutine, against a streamer
// reading from url whose stateHandler's queue is full of STATE messages with
// no peer. It returns the queue and a channel that gets what stream returns.
func streamWithFullStateQueue(url string, idleTimeout time.Duration) (
	queue chan *ristypes.RISMessage, streamErr chan error) {
	s := New(logger.New(), metrics.New())
	s.url = url
	s.idleTimeout = idleTimeout
	s.RegisterHandler(stateHandler{})
	queue = s.handlers[0].queue
	for len(queue) < cap(queue) {
		queue <- &ristypes.RISMessage{Type: ristypes.MessageTypeState}
	}

	streamErr = make(chan error, 1)
	go func() { streamErr <- s.stream(context.Background()) }()

	return queue, streamErr
}

// TestStreamWaitsForRoomForAStateMessage has the server send one STATE message
// whose state is down while the queue of the handler that wants it is full. A
// full queue is where the random drop always drops, and where a message that
// cannot wait is dropped. The streamer must not drop it: it waits until the
// handler takes a message off its queue, then queues it.
func TestStreamWaitsForRoomForAStateMessage(t *testing.T) {
	const (
		stillWaitingAfter = 100 * time.Millisecond
		maxWait           = 5 * time.Second
	)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		_, _ = io.WriteString(w, stateDownLine)
	}))
	defer srv.Close()

	queue, streamErr := streamWithFullStateQueue(srv.URL, streamIdleTimeout)

	select {
	case err := <-streamErr:
		t.Fatalf("stream returned %v while the queue was full: it dropped the STATE message", err)
	case <-time.After(stillWaitingAfter):
	}

	<-queue // the handler takes a message, which makes room
	select {
	case err := <-streamErr:
		if !errors.Is(err, errStreamEnded) {
			t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
		}
	case <-time.After(maxWait):
		t.Fatalf("stream did not queue the STATE message within %s of there being room", maxWait)
	}

	var last *ristypes.RISMessage
	for len(queue) > 0 {
		last = <-queue
	}
	if last.Peer != stateDownPeer || last.State != ristypes.StateDown {
		t.Errorf("last queued message has peer %q, state %q; want %q, %q",
			last.Peer, last.State, stateDownPeer, ristypes.StateDown)
	}
}

// TestStreamStopsWaitingForRoomAtTheIdleTimeout has the server send one STATE
// message and then hold the response open, while the queue of the handler
// that wants it stays full. The wait for room must not stall the read loop
// past its idle timeout: the stream ends as when no line arrives.
func TestStreamStopsWaitingForRoomAtTheIdleTimeout(t *testing.T) {
	const (
		idleTimeout = 50 * time.Millisecond
		maxWait     = 5 * time.Second
	)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_, _ = io.WriteString(w, stateDownLine)
		w.(http.Flusher).Flush()
		<-r.Context().Done()
	}))
	defer srv.Close()

	_, streamErr := streamWithFullStateQueue(srv.URL, idleTimeout)

	select {
	case err := <-streamErr:
		if err == nil || errors.Is(err, errStreamEnded) {
			t.Errorf("stream returned %v, want the error of a stream dropped as idle", err)
		}
	case <-time.After(maxWait):
		t.Fatalf("stream still waiting for room %s after the idle timeout of %s", maxWait, idleTimeout)
	}
}

// TestStopBeforeMessageReachesQueues stops the streamer after the read loop
// has checked for cancellation but before it hands the message to the handler
// queues. That is the gap a stop from another goroutine can land in, and it
// used to end in "send on closed channel". The raw handler runs in that gap on
// the read loop itself, so calling Stop from it hits the gap every time.
func TestStopBeforeMessageReachesQueues(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		_, _ = io.WriteString(w, updateLine)
	}))
	defer srv.Close()

	s := New(logger.New(), metrics.New())
	s.url = srv.URL
	s.RegisterHandler(updateHandler{})
	s.RegisterRawHandler(func(string) { s.Stop() })

	// Start would run the stream in the background, where the test cannot
	// wait for it. Setting cancel as Start does lets Stop cancel the stream
	// run here instead.
	ctx, cancel := context.WithCancel(context.Background())
	s.cancel = cancel

	if err := s.stream(ctx); !errors.Is(err, context.Canceled) {
		t.Fatalf("stream returned %v, want %v", err, context.Canceled)
	}

	// A second Stop must not close the queues again.
	s.Stop()
}

// TestReconnectWhenStreamEnds has the server end the first response
// normally, as when RIS Live restarts. The streamer must treat that as a
// disconnect and connect again.
func TestReconnectWhenStreamEnds(t *testing.T) {
	endResponse := func(http.ResponseWriter, *http.Request) {}
	checkReconnect(t, endResponse, streamIdleTimeout)
}

// TestReconnectWhenStreamGoesIdle has the server hold the first response
// open without sending anything. The streamer must drop it once the idle
// timeout passes and connect again.
func TestReconnectWhenStreamGoesIdle(t *testing.T) {
	const idleTimeout = 50 * time.Millisecond
	holdResponse := func(w http.ResponseWriter, r *http.Request) {
		w.(http.Flusher).Flush()
		<-r.Context().Done()
	}
	checkReconnect(t, holdResponse, idleTimeout)
}

// TestReconnectWhenServerNeverAnswers has the server accept the first
// request and never answer it, not even with headers. The streamer must give
// up on it once the idle timeout passes and connect again.
func TestReconnectWhenServerNeverAnswers(t *testing.T) {
	const idleTimeout = 50 * time.Millisecond
	neverAnswer := func(_ http.ResponseWriter, r *http.Request) {
		<-r.Context().Done()
	}
	checkReconnect(t, neverAnswer, idleTimeout)
}

// TestStreamNotDroppedWhileLinesArrive has the server send lines well within
// the idle timeout for several times the timeout, then end the response. Each
// line must start the idle timeout over, so the stream lasts until the server
// ends it instead of being dropped as idle.
func TestStreamNotDroppedWhileLinesArrive(t *testing.T) {
	const (
		idleTimeout  = 100 * time.Millisecond
		lineInterval = idleTimeout / 10
		sendFor      = 5 * idleTimeout
	)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		deadline := time.Now().Add(sendFor)
		for time.Now().Before(deadline) {
			_, _ = io.WriteString(w, updateLine)
			w.(http.Flusher).Flush()
			time.Sleep(lineInterval)
		}
	}))
	defer srv.Close()

	s := New(logger.New(), metrics.New())
	s.url = srv.URL
	s.idleTimeout = idleTimeout

	if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) {
		t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
	}
}

// checkReconnect runs a streamer against a fake RIS Live server. The first
// response is served by firstResponse; every later one carries one UPDATE
// line and then stays open. The streamer must come back for a second
// response, must not be marked connected when it does, and must handle the
// UPDATE in it.
func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration) {
	t.Helper()

	const maxWait = 5 * time.Second

	tracker := metrics.New()
	connectedAtReconnect := make(chan bool, 1)
	var requests atomic.Int32
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if requests.Add(1) == 1 {
			firstResponse(w, r)

			return
		}
		select {
		case connectedAtReconnect <- tracker.IsConnected():
		default:
		}
		_, _ = io.WriteString(w, updateLine)
		w.(http.Flusher).Flush()
		<-r.Context().Done()
	}))
	defer srv.Close()

	s := New(logger.New(), tracker)
	s.url = srv.URL
	s.idleTimeout = idleTimeout
	s.minBackoff = time.Millisecond
	handler := updateHandler{handled: make(chan struct{}, 1)}
	s.RegisterHandler(handler)
	if err := s.Start(); err != nil {
		t.Fatalf("Start returned error: %v", err)
	}
	defer s.Stop()

	select {
	case <-handler.handled:
	case <-time.After(maxWait):
		t.Fatalf("no message from a second response handled within %s", maxWait)
	}

	if <-connectedAtReconnect {
		t.Error("streamer was still marked connected when it came back for a second response")
	}
}

// settledGoroutineCount lets transient goroutines finish, then reports the
// current count.
func settledGoroutineCount() int {
	prev := runtime.NumGoroutine()
	for range 20 {
		time.Sleep(10 * time.Millisecond)
		cur := runtime.NumGoroutine()
		if cur == prev {
			return cur
		}
		prev = cur
	}

	return prev
}

// waitForGoroutines waits until the goroutine count drops to target or below.
func waitForGoroutines(target int) bool {
	for range 100 {
		if runtime.NumGoroutine() <= target {
			return true
		}
		time.Sleep(10 * time.Millisecond)
	}

	return false
}