{{range .Origins}}
+ {{if .ASN}}
AS{{.ASN}}
{{if .Handle}} ({{.Handle}}){{end}}
{{if .Description}} {{.Description}}{{end}}
+ {{else}}
+ No known origin
+ {{end}}
{{.PeerCount}} peer{{if ne .PeerCount 1}}s{{end}}
{{end}}
@@ -312,7 +316,11 @@
{{range .Routes}}
|
+ {{if .OriginASN}}
AS{{.OriginASN}}
+ {{else}}
+ No known origin
+ {{end}}
|
{{.PeerIP}} |
{{range $i, $as := .ASPathWithHandle}}{{if $i}} → {{end}}{{if $as.Handle}}{{$as.Handle}}{{else}}AS{{$as.ASN}}{{end}}{{end}} |
diff --git a/internal/templates/prefix_length.html b/internal/templates/prefix_length.html
index 54955b5..b22e810 100644
--- a/internal/templates/prefix_length.html
+++ b/internal/templates/prefix_length.html
@@ -172,9 +172,13 @@
{{ .Prefix }} |
{{ .Age }} |
+ {{ if .OriginASN }}
AS{{ .OriginASN }}{{ if .OriginASDescription }} ({{ .OriginASDescription }}){{ end }}
+ {{ else }}
+ No known origin
+ {{ end }}
|
{{ end }}
--
2.54.0
From 28df9f8d49e0d0194f3bb06de3854d5391f56c73 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Mon, 19 Oct 2026 13:08:48 +0200
Subject: [PATCH 18/45] Drain the handler queues before stopping the handlers
on shutdown (closes #69)
Shutdown stopped the handlers before the streamer, so updates still in the
handler queues, up to 20,000 per queue, were not written, and a lost
withdrawal left its route in the table after a restart.
Shutdown now stops the streamer first. Streamer.Stop queues the STATE
messages it holds for a full queue, closes the queues and returns once
every worker has handed its handler everything on its queue; then each
handler writes what it holds before its Stop returns. PeeringHandler.Stop
now waits for its loop's final write instead of running a second one
beside it.
Model: opus-5-5
---
internal/routewatch/app.go | 8 +-
.../app_integration_internal_test.go | 125 ++++++++++++++++++
internal/routewatch/peeringhandler.go | 16 ++-
.../peeringhandler_internal_test.go | 16 +++
internal/streamer/streamer.go | 44 ++++--
internal/streamer/streamer_internal_test.go | 52 +++++++-
6 files changed, 235 insertions(+), 26 deletions(-)
diff --git a/internal/routewatch/app.go b/internal/routewatch/app.go
index 9e79f11..3aeb1e3 100644
--- a/internal/routewatch/app.go
+++ b/internal/routewatch/app.go
@@ -149,7 +149,10 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
rw.shutdown = true
rw.mu.Unlock()
- // Stop batched handlers first to flush remaining batches
+ // Stop the streamer first: it returns once the handlers have been handed
+ // every message it queued. Then each handler writes what it holds.
+ rw.streamer.Stop()
+
if rw.dbHandler != nil {
rw.logger.Info("Flushing database handler")
rw.dbHandler.Stop()
@@ -180,9 +183,6 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
rw.dbMaintainer.Stop()
}
- // Stop services
- rw.streamer.Stop()
-
// Stop HTTP server with a timeout of its own, however much of ctx is left
const serverStopTimeout = 5 * time.Second
diff --git a/internal/routewatch/app_integration_internal_test.go b/internal/routewatch/app_integration_internal_test.go
index 13bb7aa..bdab0a4 100644
--- a/internal/routewatch/app_integration_internal_test.go
+++ b/internal/routewatch/app_integration_internal_test.go
@@ -3,6 +3,9 @@ package routewatch
import (
"context"
"fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
"strings"
"sync"
"testing"
@@ -12,6 +15,7 @@ import (
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/metrics"
+ "git.eeqj.de/sneak/routewatch/internal/ristypes"
"git.eeqj.de/sneak/routewatch/internal/server"
"git.eeqj.de/sneak/routewatch/internal/streamer"
"github.com/google/uuid"
@@ -529,3 +533,124 @@ func TestRouteWatchLiveFeed(t *testing.T) {
t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings)
}
+
+// gatedPrefixHandler is a PrefixHandler whose worker in the streamer waits on
+// the first message until release is closed, so the messages after it stay on
+// the streamer's queue.
+type gatedPrefixHandler struct {
+ *PrefixHandler
+
+ release chan struct{}
+}
+
+func (h gatedPrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
+ <-h.release
+ h.PrefixHandler.HandleMessage(msg)
+}
+
+// announcementFeed returns the lines RIS Live would send for one UPDATE per
+// prefix, each announcing that prefix from testPeerIP with origin testASNA,
+// followed by a KEEPALIVE. The streamer hands a KEEPALIVE to no handler, so
+// once a raw handler has seen it, every update before it is queued.
+func announcementFeed(prefixes []string) string {
+ var feed strings.Builder
+
+ for _, prefix := range prefixes {
+ fmt.Fprintf(&feed, `{"type":"ris_message","data":{"type":"UPDATE",`+
+ `"peer":"%s","peer_asn":"%d","timestamp":1700000000,"path":[%d],`+
+ `"announcements":[{"next_hop":"%s","prefixes":["%s"]}]}}`+"\n",
+ testPeerIP, testASNA, testASNA, testPeerIP, prefix)
+ }
+
+ fmt.Fprintf(&feed, `{"type":"ris_message","data":{"type":"KEEPALIVE",`+
+ `"peer":"%s","peer_asn":"%d","timestamp":1700000000}}`+"\n",
+ testPeerIP, testASNA)
+
+ return feed.String()
+}
+
+// TestShutdownWritesTheUpdatesQueuedInTheStreamer has a local server send
+// announcements of three prefixes and keep the response open. The prefix
+// handler's worker waits on the first one until the server sees the streamer
+// close the connection, which only Shutdown does, so all three are still in
+// the streamer when Shutdown starts. Each prefix must be in the database when
+// Shutdown returns.
+func TestShutdownWritesTheUpdatesQueuedInTheStreamer(t *testing.T) {
+ t.Parallel()
+
+ const maxWait = 10 * time.Second
+
+ prefixes := []string{"192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24"}
+ feed := announcementFeed(prefixes)
+
+ release := make(chan struct{})
+
+ srv := httptest.NewServer(http.HandlerFunc(
+ func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.WriteString(w, feed)
+ _ = http.NewResponseController(w).Flush()
+
+ <-r.Context().Done()
+ close(release)
+ }))
+ defer srv.Close()
+
+ store := newMockStore()
+ log := logger.New()
+ cfg := &config.Config{}
+ s := streamer.New(log, metrics.New())
+ s.SetURL(srv.URL)
+
+ rw := New(Dependencies{
+ DB: store,
+ Streamer: s,
+ Server: server.New(store, s, log, cfg),
+ Logger: log,
+ Config: cfg,
+ })
+ rw.prefixHandler = NewPrefixHandler(store, log)
+ s.RegisterHandler(gatedPrefixHandler{
+ PrefixHandler: rw.prefixHandler, release: release,
+ })
+
+ linesSeen := make(chan struct{}, len(prefixes)+1)
+
+ s.RegisterRawHandler(func(string) { linesSeen <- struct{}{} })
+
+ err := s.Start(t.Context())
+ if err != nil {
+ t.Fatalf("Start returned error: %v", err)
+ }
+
+ for range cap(linesSeen) {
+ select {
+ case <-linesSeen:
+ case <-time.After(maxWait):
+ t.Fatalf("the streamer did not read %d lines within %s",
+ cap(linesSeen), maxWait)
+ }
+ }
+
+ shutDown := make(chan struct{})
+
+ go func() {
+ rw.Shutdown(t.Context())
+ close(shutDown)
+ }()
+
+ select {
+ case <-shutDown:
+ case <-time.After(maxWait):
+ t.Fatalf("Shutdown did not return within %s", maxWait)
+ }
+
+ store.mu.Lock()
+ defer store.mu.Unlock()
+
+ for _, prefix := range prefixes {
+ if _, ok := store.Prefixes[prefix]; !ok {
+ t.Errorf("%s was queued in the streamer when Shutdown started "+
+ "but is not in the database", prefix)
+ }
+ }
+}
diff --git a/internal/routewatch/peeringhandler.go b/internal/routewatch/peeringhandler.go
index 65ce19a..d30d06b 100644
--- a/internal/routewatch/peeringhandler.go
+++ b/internal/routewatch/peeringhandler.go
@@ -47,6 +47,7 @@ type PeeringHandler struct {
droppedPaths int // paths dropped because the map was full
stopCh chan struct{}
+ wg sync.WaitGroup
}
// NewPeeringHandler creates and initializes a new PeeringHandler with the
@@ -62,6 +63,8 @@ func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler
}
// Start the periodic processing goroutine
+ h.wg.Add(1)
+
go h.processLoop()
return h
@@ -130,17 +133,18 @@ func (h *PeeringHandler) ProcessPeeringsNow() {
}
// Stop gracefully shuts down the handler by signaling the background
-// goroutine to stop and performing a final synchronous processing of
-// any remaining AS paths. This ensures no peering data is lost during
-// shutdown.
+// goroutine to stop and waiting while it processes the remaining AS paths,
+// so no peering data is lost during shutdown.
func (h *PeeringHandler) Stop() {
close(h.stopCh)
- // Process any remaining peerings synchronously
- h.processPeerings()
+ h.wg.Wait()
}
-// processLoop runs periodically to process AS paths into peerings
+// processLoop runs periodically to process AS paths into peerings, and once
+// more on Stop.
func (h *PeeringHandler) processLoop() {
+ defer h.wg.Done()
+
ticker := time.NewTicker(peeringProcessInterval)
defer ticker.Stop()
diff --git a/internal/routewatch/peeringhandler_internal_test.go b/internal/routewatch/peeringhandler_internal_test.go
index 32da7e0..13c5d66 100644
--- a/internal/routewatch/peeringhandler_internal_test.go
+++ b/internal/routewatch/peeringhandler_internal_test.go
@@ -188,6 +188,22 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) {
}
}
+// TestPeeringHandlerStopWritesTheRemainingPaths hands a running handler one
+// path and stops it. The peering must be in the database when Stop returns.
+func TestPeeringHandlerStopWritesTheRemainingPaths(t *testing.T) {
+ t.Parallel()
+
+ store := &recordingStore{mockStore: newMockStore()}
+ h := NewPeeringHandler(store, logger.New())
+
+ handle(h, time.Now().UTC(), testASNA, testASNB)
+ h.Stop()
+
+ if got := store.callCount(); got != 1 {
+ t.Errorf("want 1 RecordPeering call when Stop returns, got %d", got)
+ }
+}
+
// TestPeeringHandlerCapDropsAndCounts verifies that a full map drops new paths
// and counts them, while a path already tracked is refreshed rather than
// dropped.
diff --git a/internal/streamer/streamer.go b/internal/streamer/streamer.go
index e54412c..698768f 100644
--- a/internal/streamer/streamer.go
+++ b/internal/streamer/streamer.go
@@ -120,7 +120,7 @@ type handlerInfo struct {
queue chan *ristypes.RISMessage
metrics handlerMetrics
// heldState holds, in feed order, the STATE messages that found queue
- // full (see queueMessage). Only the read loop uses it.
+ // full (see queueMessage). Only the read loop uses it, and then Stop.
heldState []*ristypes.RISMessage
}
@@ -141,6 +141,7 @@ type Streamer struct {
mu sync.RWMutex
cancel context.CancelFunc
running bool
+ workers sync.WaitGroup // the handler workers; Stop waits for them
metrics *metrics.Tracker
totalDropped atomic.Uint64 // Total dropped messages across all handlers
random *rand.Rand // Random number generator for backpressure drops
@@ -194,10 +195,18 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) {
// If we're already running, start a worker for this handler
if s.running {
+ s.workers.Add(1)
+
go s.runHandlerWorker(info)
}
}
+// SetURL makes the streamer read from url instead of RIS Live. Call it before
+// Start. Tests use it to read from a local server.
+func (s *Streamer) SetURL(url string) {
+ s.url = url
+}
+
// RegisterRawHandler sets a callback to receive raw JSON lines from the stream
// before they are parsed. Only one raw handler can be registered at a time;
// subsequent calls will replace the previous handler.
@@ -227,6 +236,8 @@ func (s *Streamer) Start(ctx context.Context) error {
// Start workers for each handler
for _, info := range s.handlers {
+ s.workers.Add(1)
+
go s.runHandlerWorker(info)
}
@@ -240,9 +251,10 @@ func (s *Streamer) Start(ctx context.Context) error {
return nil
}
-// Stop halts the streaming connection and shuts down all handler workers.
-// It cancels the streaming context, closes all handler queues, and updates
-// the connection status in metrics. This method is safe to call multiple times.
+// Stop ends the stream and stops the handler workers. It returns once each
+// worker has handed its handler every message left on its queue and then the
+// STATE messages held for it (see queueMessage). This method is safe to call
+// multiple times.
func (s *Streamer) Stop() {
s.mu.Lock()
if s.cancel == nil {
@@ -252,16 +264,28 @@ func (s *Streamer) Stop() {
return
}
+ // Once ctx is cancelled the read loop queues and holds nothing more (see
+ // dispatchMessage).
s.cancel()
s.cancel = nil
- // Close all handler queues to signal workers to stop
+
for _, info := range s.handlers {
+ // The held messages came after everything on the queue. Each waits
+ // for the worker, which does not take s.mu, to make room.
+ for _, msg := range info.heldState {
+ info.queue <- msg
+ }
+
+ info.heldState = nil
+
+ // The worker returns once it has emptied the closed queue.
close(info.queue)
}
s.running = false
s.mu.Unlock()
s.metrics.SetConnected(false)
+ s.workers.Wait()
}
// IsRunning reports whether the streamer is currently connected and processing
@@ -354,6 +378,8 @@ func (s *Streamer) GetDroppedMessages() uint64 {
// runHandlerWorker processes messages for a specific handler
func (s *Streamer) runHandlerWorker(info *handlerInfo) {
+ defer s.workers.Done()
+
for msg := range info.queue {
start := time.Now()
@@ -824,16 +850,16 @@ func (s *Streamer) dispatchMessage(
}
// queueMessage puts msg on the handler's queue and reports false if it dropped
-// msg instead. It never waits for room, so a full queue holds up neither the
-// read loop nor Stop.
+// msg instead. It never waits for room, so a full queue does not hold up the
+// read loop.
//
// A message is dropped while the queue is filling up (shouldDropForBackpressure)
// or full, except a STATE message: one whose state is down is the only notice
// that all of a peer's routes are gone. A STATE message skips the random drop,
// and one that finds the queue full is held in info.heldState. Held messages
// go into the queue, oldest first, before anything else for the handler, as
-// soon as there is room; while any is still held, the handler's other messages
-// are dropped, so none of them overtakes a held down.
+// soon as there is room, or when Stop is called; while any is still held, the
+// handler's other messages are dropped, so none of them overtakes a held down.
//
// The held list stays small: a STATE message is a few hundred bytes and the
// feed sends about ten a second, so a queue full for a minute holds about 600.
diff --git a/internal/streamer/streamer_internal_test.go b/internal/streamer/streamer_internal_test.go
index e56ff91..308044a 100644
--- a/internal/streamer/streamer_internal_test.go
+++ b/internal/streamer/streamer_internal_test.go
@@ -255,6 +255,24 @@ func TestHeldStateMessagesKeepTheirPlaceInTheQueue(t *testing.T) {
}
}
+// takeAll takes the messages off queue, as a handler's worker would, and sends
+// them on the returned channel once queue is closed.
+func takeAll(queue chan *ristypes.RISMessage) chan []*ristypes.RISMessage {
+ taken := make(chan []*ristypes.RISMessage, 1)
+
+ go func() {
+ var msgs []*ristypes.RISMessage
+
+ for msg := range queue {
+ msgs = append(msgs, msg)
+ }
+
+ taken <- msgs
+ }()
+
+ return taken
+}
+
// newStreamerWithFullStateQueue returns a streamer reading from url with one
// handler, a stateHandler whose queue is full.
func newStreamerWithFullStateQueue(url string) *Streamer {
@@ -306,20 +324,23 @@ func TestOtherHandlersKeepReceivingWhileAStateMessageIsHeld(t *testing.T) {
}
}
-// TestStopReturnsAtOnceWithStateMessagesHeld has the server send two STATE
-// messages and hold the response open while the queue of the handler that
-// wants them is full. Stop must return at once and end the stream.
-func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) {
+// TestStopQueuesTheHeldStateMessages has the server send two STATE messages
+// and an UPDATE and hold the response open while the queue of the handler that
+// wants STATE messages is full, so both are held. Stop must put both on the
+// queue, after the messages already on it, before it closes the queue, and
+// must end the stream.
+func TestStopQueuesTheHeldStateMessages(t *testing.T) {
t.Parallel()
const (
- lines = 2
+ held = 2
+ lines = held + 1
maxWait = 5 * time.Second
)
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
- _, _ = io.WriteString(w, stateDownLine+stateDownLine)
+ _, _ = io.WriteString(w, stateDownLine+stateDownLine+updateLine)
flush(w)
<-r.Context().Done()
@@ -328,7 +349,7 @@ func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) {
s := newStreamerWithFullStateQueue(srv.URL)
// The raw handler sees each line before the read loop hands it to the
- // queues, so once it has seen the second line, the first is held.
+ // queues, so once it has seen the UPDATE, both STATE messages are held.
linesSeen := make(chan struct{}, lines)
s.RegisterRawHandler(func(string) { linesSeen <- struct{}{} })
@@ -350,6 +371,9 @@ func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) {
}
}
+ // No worker runs here: the test takes from the full queue in its place.
+ queue := s.handlers[0].queue
+ taken := takeAll(queue)
stopped := make(chan struct{})
go func() {
@@ -364,6 +388,20 @@ func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) {
maxWait)
}
+ // Stop has closed the queue, so takeAll is done or about to be.
+ msgs := <-taken
+ if len(msgs) != cap(queue)+held {
+ t.Fatalf("%d messages came off the queue, want the %d on it and the "+
+ "%d held", len(msgs), cap(queue), held)
+ }
+
+ for _, msg := range msgs[cap(queue):] {
+ if msg.Peer != stateDownPeer || msg.State != ristypes.StateDown {
+ t.Errorf("message after those on the queue has peer %q, state %q; "+
+ "want the down from %s", msg.Peer, msg.State, stateDownPeer)
+ }
+ }
+
select {
case err := <-streamErr:
if !errors.Is(err, context.Canceled) {
--
2.54.0
From 5806ddaef7d19bb7d16b1f2868d0bd63a4de29c4 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Mon, 19 Oct 2026 13:26:36 +0200
Subject: [PATCH 19/45] Run database maintenance under the database lock
(closes #94)
The database maintainer ran Vacuum and Analyze without the database lock
every batch write holds, so on the live feed every incremental vacuum
failed with "database is locked", and an hourly ANALYZE made a batch write
fail and lose its updates. Vacuum, Analyze and Checkpoint now take that
lock, so maintenance and batch writes wait for each other. Analyze runs
PRAGMA optimize with an analysis limit, which analyzes only tables whose
size changed a lot, so it holds the lock briefly however large the file.
Model: opus-5-5
---
internal/database/database.go | 58 ++++-
internal/database/database_internal_test.go | 234 +++++++++++++++++++-
2 files changed, 272 insertions(+), 20 deletions(-)
diff --git a/internal/database/database.go b/internal/database/database.go
index 248acee..a536e77 100644
--- a/internal/database/database.go
+++ b/internal/database/database.go
@@ -68,6 +68,18 @@ const (
sqliteHardHeapLimitBytes = 1610612736
)
+// Settings of the ANALYZE that Analyze runs.
+const (
+ // analysisLimit is about how many rows of each index ANALYZE reads
+ // (PRAGMA analysis_limit); SQLite estimates the rest.
+ analysisLimit = 1000
+ // optimizeAllTables is the PRAGMA optimize mask that runs ANALYZE (0x2)
+ // on every table that has an index without statistics or whose size has
+ // changed tenfold since it was last analyzed, not only on the tables this
+ // connection has queried (0x10000).
+ optimizeAllTables = 0x10002
+)
+
// Bounds on the work GetRandomPrefixesByLength does to pick its sample.
const (
// maxPrefixesToShuffle is the most prefixes of one mask length that are
@@ -182,7 +194,6 @@ func (d *Database) Initialize() error {
// enough for the whole pool.
pragmas := []string{
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
- "PRAGMA analysis_limit=0", // Disable automatic ANALYZE
fmt.Sprintf("PRAGMA soft_heap_limit=%d", sqliteSoftHeapLimitBytes),
fmt.Sprintf("PRAGMA hard_heap_limit=%d", sqliteHardHeapLimitBytes),
}
@@ -2151,14 +2162,18 @@ func (d *Database) GetIPInfoContext(
return info, nil
}
-// Vacuum runs incremental vacuum to reclaim unused pages without blocking
-// writes. It frees up to the specified number of pages per call (0 = all
-// freeable pages).
+// Vacuum runs incremental vacuum to return up to 1000 unused pages to the
+// filesystem. Like Analyze and Checkpoint, it holds the database lock, so it
+// and the batch writes wait for each other instead of failing on SQLite's
+// write lock.
func (d *Database) Vacuum(ctx context.Context) error {
// Free up to 1000 pages per call (~4MB with default 4KB page size)
- // This keeps each vacuum operation quick and non-blocking
+ // This keeps each vacuum operation quick
const pagesToFree = 1000
+ d.lock("Vacuum")
+ defer d.unlock()
+
rows, err := d.db.QueryContext(ctx,
fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree))
if err != nil {
@@ -2180,9 +2195,20 @@ func (d *Database) Vacuum(ctx context.Context) error {
return nil
}
-// Analyze runs the SQLite ANALYZE command to update query planner statistics.
+// Analyze updates the query planner statistics. It runs ANALYZE, through
+// PRAGMA optimize, only on the tables that have an index without statistics or
+// whose size has changed tenfold since they were last analyzed: even with
+// analysisLimit, ANALYZE counts every entry of each index, so it takes longer
+// as the tables grow. It holds the database lock.
func (d *Database) Analyze(ctx context.Context) error {
- _, err := d.db.ExecContext(ctx, "ANALYZE")
+ d.lock("Analyze")
+ defer d.unlock()
+
+ // analysis_limit is a setting of the connection, so it is set in the same
+ // call as PRAGMA optimize, which runs both statements on one connection.
+ _, err := d.db.ExecContext(ctx, fmt.Sprintf(
+ "PRAGMA analysis_limit=%d; PRAGMA optimize=%#x",
+ analysisLimit, optimizeAllTables))
if err != nil {
return fmt.Errorf("failed to analyze database: %w", err)
}
@@ -2192,8 +2218,11 @@ func (d *Database) Analyze(ctx context.Context) error {
// Checkpoint runs a WAL checkpoint to transfer data from the WAL to the main database.
// Uses TRUNCATE mode which blocks writers briefly but ensures complete checkpoint,
-// keeping the WAL small for fast read performance.
+// keeping the WAL small for fast read performance. It holds the database lock.
func (d *Database) Checkpoint(ctx context.Context) error {
+ d.lock("Checkpoint")
+ defer d.unlock()
+
_, err := d.db.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
if err != nil {
return fmt.Errorf("failed to checkpoint WAL: %w", err)
@@ -2214,21 +2243,26 @@ func (d *Database) Ping(ctx context.Context) error {
return nil
}
+// The debug messages lock logs, with the operation's name, when it starts to
+// wait for the database mutex and when it holds it.
+const (
+ lockWaitMessage = "Acquiring database lock"
+ lockHeldMessage = "Database lock acquired"
+)
+
// lock acquires the database mutex and logs debug information
func (d *Database) lock(operation string) {
// Get caller information
_, file, line, _ := runtime.Caller(1)
caller := fmt.Sprintf("%s:%d", filepath.Base(file), line)
- d.logger.Debug("Acquiring database lock",
- "operation", operation, "caller", caller)
+ d.logger.Debug(lockWaitMessage, "operation", operation, "caller", caller)
d.mu.Lock()
d.lockedAt = time.Now()
d.lockedBy = fmt.Sprintf("%s (%s)", operation, caller)
- d.logger.Debug("Database lock acquired",
- "operation", operation, "caller", caller)
+ d.logger.Debug(lockHeldMessage, "operation", operation, "caller", caller)
}
// unlock releases the database mutex and logs debug information including
diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go
index f2b2e63..d93b188 100644
--- a/internal/database/database_internal_test.go
+++ b/internal/database/database_internal_test.go
@@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"fmt"
+ "log/slog"
"net/netip"
"slices"
"sync"
@@ -47,6 +48,23 @@ const (
asnSecondBand = 100
)
+// Parameters for the tests of maintenance and the database lock.
+const (
+ // lockMessageBuffer is how many lock messages a test database's logger
+ // holds before logging waits for the test to read them.
+ lockMessageBuffer = 64
+ // operationKey is the log attribute that names the operation taking the
+ // lock.
+ operationKey = "operation"
+ // testBatchWrite is the operation name the tests take the lock under.
+ testBatchWrite = "test batch write"
+ // lockWaitTimeout is the context timeout of a maintenance call that the
+ // tests let run out while it waits for the lock.
+ lockWaitTimeout = time.Millisecond
+ // lockTestASN is the AS the batch write in the lock tests stores.
+ lockTestASN = 64500
+)
+
// Parameters for the prefix-length sample tests.
const (
// sampleLimit is how many prefixes the tests ask the sample for.
@@ -474,14 +492,10 @@ func TestVacuumReturnsFreePages(t *testing.T) {
}
}
-// TestBatchWriteDuringCheckpoint reproduces issue #25. A batch write reads
-// (SELECT) before it writes (INSERT/UPDATE). Under the default deferred locking
-// the transaction begins as a reader and, when the maintainer's WAL checkpoint
-// holds the write lock, its upgrade to writer fails immediately with "database
-// is locked" without honouring busy_timeout, dropping the batch. With
-// _txlock=immediate the transaction takes the write lock at BEGIN and waits, so
-// no batch is dropped. The checkpoint runs without the Database mutex, exactly
-// as the background maintainer does in production.
+// TestBatchWriteDuringCheckpoint runs checkpoints back to back while batch
+// writes run and checks that every batch write succeeds: a checkpoint holds the
+// database lock as a batch write does, and checkpoints must not keep batch
+// writes from getting it.
func TestBatchWriteDuringCheckpoint(t *testing.T) {
t.Parallel()
@@ -530,6 +544,210 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) {
wg.Wait()
}
+// lockMessage is a debug message the database lock logged, and the name of the
+// operation that logged it.
+type lockMessage struct {
+ text string
+ operation string
+}
+
+// lockMessages is a log handler that passes on each message the database lock
+// logs. The lock tests read it to know that an operation waits for the lock or
+// holds it, instead of guessing how long that takes.
+type lockMessages chan lockMessage
+
+// Enabled lets every message through, so the lock's debug messages arrive.
+func (m lockMessages) Enabled(context.Context, slog.Level) bool { return true }
+
+// Handle passes on a message that names an operation and drops the others.
+func (m lockMessages) Handle(_ context.Context, record slog.Record) error {
+ record.Attrs(func(attr slog.Attr) bool {
+ if attr.Key != operationKey {
+ return true
+ }
+
+ m <- lockMessage{text: record.Message, operation: attr.Value.String()}
+
+ return false
+ })
+
+ return nil
+}
+
+// WithAttrs returns m: the tests need no attributes added.
+func (m lockMessages) WithAttrs([]slog.Attr) slog.Handler { return m }
+
+// WithGroup returns m: the tests need no groups.
+func (m lockMessages) WithGroup(string) slog.Handler { return m }
+
+// newDatabaseWithLockMessages opens a new database whose logger passes the lock's
+// messages to the returned channel.
+func newDatabaseWithLockMessages(t *testing.T) (*Database, lockMessages) {
+ t.Helper()
+
+ messages := make(lockMessages, lockMessageBuffer)
+ cfg := &config.Config{StateDir: t.TempDir()}
+
+ db, err := New(cfg, &logger.Logger{Logger: slog.New(messages)})
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+
+ t.Cleanup(func() { _ = db.Close() })
+
+ return db, messages
+}
+
+// waitForLockMessage waits until operation logs text, and fails the test if
+// returned delivers first: the call running operation returned without logging
+// text.
+func waitForLockMessage(
+ t *testing.T,
+ messages lockMessages,
+ text, operation string,
+ returned <-chan error,
+) {
+ t.Helper()
+
+ for {
+ select {
+ case message := <-messages:
+ if message.text == text && message.operation == operation {
+ return
+ }
+ case err := <-returned:
+ t.Fatalf("%s returned (error: %v) without logging %q",
+ operation, err, text)
+ }
+ }
+}
+
+// TestVacuumWaitsForBatchWrite holds the database lock as a batch write does and
+// checks that Vacuum waits for it and then succeeds. Vacuum ran without the
+// lock, met the batch writes at SQLite's write lock and failed with "database
+// is locked" (https://git.eeqj.de/sneak/routewatch/issues/94).
+func TestVacuumWaitsForBatchWrite(t *testing.T) {
+ t.Parallel()
+
+ db, messages := newDatabaseWithLockMessages(t)
+
+ db.lock(testBatchWrite)
+
+ vacuumed := make(chan error, 1)
+
+ go func() { vacuumed <- db.Vacuum(context.Background()) }()
+
+ waitForLockMessage(t, messages, lockWaitMessage, "Vacuum", vacuumed)
+
+ db.unlock()
+
+ err := <-vacuumed
+ if err != nil {
+ t.Fatalf("Vacuum after the batch write: %v", err)
+ }
+}
+
+// TestBatchWriteWaitsForAnalyze keeps Analyze waiting for SQLite's write lock,
+// which another connection holds, and checks that a batch write started
+// meanwhile waits for Analyze to give up the database lock, that both then
+// succeed, and that Analyze stored statistics. ANALYZE ran without the database
+// lock, and a batch write that waited longer than the busy timeout for it failed
+// with "database is locked".
+func TestBatchWriteWaitsForAnalyze(t *testing.T) {
+ t.Parallel()
+
+ db, messages := newDatabaseWithLockMessages(t)
+ ctx := context.Background()
+ asns := map[int]time.Time{lockTestASN: time.Now().UTC()}
+
+ // A stored AS gives ANALYZE a table to analyze and statistics to write.
+ err := db.GetOrCreateASNBatch(asns)
+ if err != nil {
+ t.Fatalf("failed to store an AS: %v", err)
+ }
+
+ // With _txlock=immediate this transaction takes SQLite's write lock at
+ // BEGIN, so ANALYZE has to wait for it while Analyze holds the database lock.
+ writer, err := db.db.BeginTx(ctx, nil)
+ if err != nil {
+ t.Fatalf("failed to begin the transaction ANALYZE waits for: %v", err)
+ }
+
+ defer func() { _ = writer.Rollback() }()
+
+ analyzed := make(chan error, 1)
+
+ go func() { analyzed <- db.Analyze(ctx) }()
+
+ waitForLockMessage(t, messages, lockHeldMessage, "Analyze", analyzed)
+
+ written := make(chan error, 1)
+
+ go func() { written <- db.GetOrCreateASNBatch(asns) }()
+
+ waitForLockMessage(t, messages, lockWaitMessage, "GetOrCreateASNBatch", written)
+
+ err = writer.Rollback()
+ if err != nil {
+ t.Fatalf("failed to end the transaction ANALYZE waits for: %v", err)
+ }
+
+ err = <-analyzed
+ if err != nil {
+ t.Errorf("Analyze: %v", err)
+ }
+
+ err = <-written
+ if err != nil {
+ t.Errorf("batch write after Analyze: %v", err)
+ }
+
+ var statistics int
+
+ err = db.db.QueryRowContext(ctx,
+ "SELECT count(*) FROM sqlite_stat1 WHERE tbl = 'asns'").Scan(&statistics)
+ if err != nil {
+ t.Fatalf("failed to read the statistics: %v", err)
+ }
+
+ if statistics == 0 {
+ t.Error("Analyze stored no statistics for the asns table")
+ }
+}
+
+// TestCheckpointLockWaitEndsWithContext holds the database lock while the
+// context of a waiting Checkpoint times out, and checks that Checkpoint then
+// returns the context's error and leaves the lock free.
+func TestCheckpointLockWaitEndsWithContext(t *testing.T) {
+ t.Parallel()
+
+ db, messages := newDatabaseWithLockMessages(t)
+
+ db.lock(testBatchWrite)
+
+ ctx, cancel := context.WithTimeout(context.Background(), lockWaitTimeout)
+ defer cancel()
+
+ checkpointed := make(chan error, 1)
+
+ go func() { checkpointed <- db.Checkpoint(ctx) }()
+
+ waitForLockMessage(t, messages, lockWaitMessage, "Checkpoint", checkpointed)
+ <-ctx.Done()
+ db.unlock()
+
+ err := <-checkpointed
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("Checkpoint = %v, want %v", err, context.DeadlineExceeded)
+ }
+
+ if !db.mu.TryLock() {
+ t.Fatal("Checkpoint kept the database lock after its context ended")
+ }
+
+ db.mu.Unlock()
+}
+
// TestGetRandomPrefixesByLengthListsPrefixOnce stores one prefix of each
// family learned from two peers in the same second and checks that the
// prefix-length sample lists each of them once.
--
2.54.0
From d84dd5646060673ac039bd79e3a41875da7b844e Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Mon, 19 Oct 2026 15:43:38 +0200
Subject: [PATCH 20/45] Make live route writes cheaper so the prefix handler
keeps up (closes #96)
The live route tables lose their random UUID key; the row id the
prefix-length sample draws is declared as the integer key. The prefix
tables are keyed by prefix and keep only first_seen; a batch inserts only
prefixes not stored yet. A route update sets only the columns that
change. Each kept index names the queries using it. Every write goes
through one connection kept for writing, with a 1 GiB page cache; reading
connections get 16 MiB, heap limits rise to 1.5 and 2 GiB (README
"Memory"). Batch writes go in key order. A benchmark run by hand writes
250,000 updates on 5 million routes faster than the feed's 10,000 a
second; a live run follows.
Model: opus-5-5
---
README.md | 21 +-
internal/database/counts_internal_test.go | 2 -
internal/database/database.go | 378 ++++++++-------
internal/database/database_internal_test.go | 443 +++++++++++++++++-
internal/database/models.go | 5 +-
internal/database/schema.sql | 78 +--
internal/database/slowquery.go | 4 +-
.../app_integration_internal_test.go | 13 +-
internal/routewatch/prefixhandler.go | 2 -
internal/server/handlers_internal_test.go | 15 +-
10 files changed, 708 insertions(+), 253 deletions(-)
diff --git a/README.md b/README.md
index 84a7bce..3526890 100644
--- a/README.md
+++ b/README.md
@@ -117,7 +117,7 @@ routewatch/
- Signal handlers registered for graceful shutdown
2. **Initialization** (`internal/routewatch/app.go`)
- - Database created with SQLite schema (WAL mode, 3GB cache)
+ - Database created with SQLite schema (WAL mode, page caches as in "Memory")
- Message handlers registered with the streamer
- HTTP server started on configured port
@@ -170,8 +170,8 @@ routewatch/
```sql
-- Core tables
asns(id, number, handle, description, first_seen, last_seen)
-prefixes_v4(id, prefix, mask_length, first_seen, last_seen)
-prefixes_v6(id, prefix, mask_length, first_seen, last_seen)
+prefixes_v4(prefix, first_seen)
+prefixes_v6(prefix, first_seen)
-- Live routing tables (one per IP version)
live_routes_v4(id, prefix, mask_length, origin_asn, peer_ip, as_path,
@@ -215,8 +215,12 @@ data it tracks. The image sets ceilings that keep it inside a 5 GiB container.
Budget:
- Go heap: a 1.5 GiB soft limit (`GOMEMLIMIT=1536MiB`, set in the image).
-- SQLite: at most 640 MiB of page cache across the connection pool (64 MiB per
- connection, 10 connections) and a 1.5 GiB hard heap limit for the C library.
+- SQLite: 1 GiB of page cache for the one connection every write uses, which
+ keeps the pages the route writes touch from one batch to the next, and 16 MiB
+ for each of the 9 reading connections, whose cache SQLite empties whenever
+ another connection has written: about 1.15 GiB in all. A 1.5 GiB soft heap
+ limit sits above the caches, so it does not shrink them, and a 2 GiB hard heap
+ limit caps the C library.
- glibc allocator: the SQLite C library runs on glibc `malloc`, which frees
page-cache chunks back to per-arena free lists rather than to the kernel, so
process RSS tracks the high-water mark of those arenas, not SQLite's live
@@ -226,6 +230,9 @@ Budget:
database writes are already serialized.
- About 0.2 GiB for everything else in the runtime.
+The ceilings add up to 3.7 GiB: the Go heap's 1.5 GiB, SQLite's 2 GiB and the
+0.2 GiB for the rest.
+
Run the container with a memory limit of 5 GiB and swap disabled:
```bash
@@ -244,8 +251,8 @@ What happens at each limit:
- Go soft limit: as the heap approaches `GOMEMLIMIT`, the runtime runs garbage
collection more aggressively rather than growing further.
-- SQLite: at a 1 GiB soft heap limit it recycles its page cache instead of
- allocating more; at the 1.5 GiB hard heap limit a statement fails with an
+- SQLite: at the 1.5 GiB soft heap limit it recycles its page cache instead of
+ allocating more; at the 2 GiB hard heap limit a statement fails with an
out-of-memory error, and the handler logs it and drops that batch. The process
keeps running.
- Handler queues: each of the four handler queues holds at most 20,000 messages.
diff --git a/internal/database/counts_internal_test.go b/internal/database/counts_internal_test.go
index c5decf7..bbe5577 100644
--- a/internal/database/counts_internal_test.go
+++ b/internal/database/counts_internal_test.go
@@ -29,7 +29,6 @@ func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute {
}
return &LiveRoute{
- ID: uuid.New(),
Prefix: prefix,
MaskLength: maskLength,
IPVersion: ipVersionV4,
@@ -44,7 +43,6 @@ func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute {
// mkV6Route builds an IPv6 live route.
func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute {
return &LiveRoute{
- ID: uuid.New(),
Prefix: prefix,
MaskLength: 32,
IPVersion: ipVersionV6,
diff --git a/internal/database/database.go b/internal/database/database.go
index a536e77..c4832bb 100644
--- a/internal/database/database.go
+++ b/internal/database/database.go
@@ -1,25 +1,27 @@
package database
import (
+ "cmp"
"context"
"database/sql"
_ "embed"
"encoding/json"
"errors"
"fmt"
+ "maps"
"math/rand/v2"
"net"
"net/netip"
"os"
"path/filepath"
"runtime"
+ "slices"
"sync"
"time"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/pkg/asinfo"
- "github.com/google/uuid"
_ "github.com/mattn/go-sqlite3" // CGO SQLite driver
)
@@ -52,20 +54,31 @@ const (
minLookupMaskLengthV6 = 16
)
-// SQLite memory tuning. cache_size and busy_timeout go in the DSN so every
-// pooled connection gets them; the heap limits are process-wide and set once.
+// SQLite memory tuning (README "Memory"). cache_size and busy_timeout go in the
+// DSN so every pooled connection gets them; the connection kept for writing
+// then sets its own cache size. The heap limits are process-wide and set once.
const (
- // sqliteCacheSizeKiB is the per-connection page cache; negative means KiB.
- // -65536 = 64 MiB, so at most 640 MiB across the 10-connection pool.
- sqliteCacheSizeKiB = -65536
+ // maxConns is the size of the connection pool: the connection kept for
+ // writing and 9 reading connections.
+ maxConns = 10
+ // sqliteReadCacheSizeKiB is the page cache of each reading connection;
+ // negative means KiB. -16384 = 16 MiB, at most 144 MiB for the 9. A reading
+ // connection's cache is emptied whenever another connection has written
+ // since its last read, many times a minute, so a large one is wasted.
+ sqliteReadCacheSizeKiB = -16384
+ // sqliteWriteCacheSizeKiB is the page cache of the connection kept for
+ // writing, -1048576 = 1 GiB. It keeps its pages from one write to the next,
+ // so the index pages the route writes touch stay in memory.
+ sqliteWriteCacheSizeKiB = -1048576
// sqliteBusyTimeoutMs is how long a connection waits on a locked database.
sqliteBusyTimeoutMs = 5000
- // sqliteSoftHeapLimitBytes (1 GiB) makes SQLite recycle its cache rather
- // than allocate once its C heap passes this size.
- sqliteSoftHeapLimitBytes = 1073741824
- // sqliteHardHeapLimitBytes (1.5 GiB) fails a statement with SQLITE_NOMEM
+ // sqliteSoftHeapLimitBytes (1.5 GiB) makes SQLite recycle its cache rather
+ // than allocate once its C heap passes this size. It is above the caches
+ // together, so it does not shrink them.
+ sqliteSoftHeapLimitBytes = 1610612736
+ // sqliteHardHeapLimitBytes (2 GiB) fails a statement with SQLITE_NOMEM
// instead of growing the C heap without bound.
- sqliteHardHeapLimitBytes = 1610612736
+ sqliteHardHeapLimitBytes = 2147483648
)
// Settings of the ANALYZE that Analyze runs.
@@ -106,7 +119,10 @@ var (
// Database manages the SQLite database connection and operations.
type Database struct {
- db *sql.DB
+ db *sql.DB
+ // writer is the connection kept for writing, taken from db when the
+ // database opens. Every write uses it while holding mu.
+ writer *sql.Conn
logger *logger.Logger
path string
mu sync.Mutex
@@ -144,7 +160,7 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
"file:%s?_cache_size=%d&_synchronous=OFF&_busy_timeout=%d"+
"&_auto_vacuum=incremental&_journal_mode=WAL&_txlock=immediate",
dbPath,
- sqliteCacheSizeKiB,
+ sqliteReadCacheSizeKiB,
sqliteBusyTimeoutMs,
)
@@ -153,20 +169,34 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
return nil, fmt.Errorf("failed to open database: %w", err)
}
- err = db.PingContext(context.Background())
+ ctx := context.Background()
+
+ err = db.PingContext(ctx)
if err != nil {
return nil, fmt.Errorf("failed to ping database: %w", err)
}
// Set connection pool parameters
// Multiple connections allow concurrent reads while writes are serialized
- const maxConns = 10
db.SetMaxOpenConns(maxConns)
db.SetMaxIdleConns(maxConns)
db.SetConnMaxLifetime(0)
+ writer, err := db.Conn(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open the writing connection: %w", err)
+ }
+
+ _, err = writer.ExecContext(ctx,
+ fmt.Sprintf("PRAGMA cache_size=%d", sqliteWriteCacheSizeKiB))
+ if err != nil {
+ return nil, fmt.Errorf("failed to set the writing connection's cache: %w",
+ err)
+ }
+
database := &Database{
- db: db, logger: logger, path: dbPath, counts: &liveCounts{},
+ db: db, writer: writer, logger: logger, path: dbPath,
+ counts: &liveCounts{},
}
err = database.Initialize()
@@ -188,10 +218,11 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
// Initialize creates the database schema if it doesn't exist.
func (d *Database) Initialize() error {
// Set SQLite pragmas for performance. Per-connection settings (cache_size,
- // synchronous, busy_timeout, journal_mode) live in the DSN; temp_store is
- // left at its default so DISTINCT temp B-trees spill to disk instead of C
- // heap. The heap limits below are process-wide, so setting them once here is
- // enough for the whole pool.
+ // synchronous, busy_timeout, journal_mode) live in the DSN, and New sets the
+ // writing connection's cache_size; temp_store is left at its default so
+ // DISTINCT temp B-trees spill to disk instead of C heap. The heap limits
+ // below are process-wide, so setting them once here is enough for the whole
+ // pool.
pragmas := []string{
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
fmt.Sprintf("PRAGMA soft_heap_limit=%d", sqliteSoftHeapLimitBytes),
@@ -208,7 +239,7 @@ func (d *Database) Initialize() error {
// Run WAL checkpoint on startup to consolidate any existing WAL
var walPages, checkpointedPages, movedPages int
- err := d.db.QueryRowContext(context.Background(),
+ err := d.writer.QueryRowContext(context.Background(),
"PRAGMA wal_checkpoint(TRUNCATE)",
).Scan(&walPages, &checkpointedPages, &movedPages)
if err != nil {
@@ -229,35 +260,47 @@ func (d *Database) Initialize() error {
return nil
}
-// Close closes the database connection.
+// Close closes the writing connection and then the connection pool.
func (d *Database) Close() error {
- return d.db.Close()
+ return errors.Join(d.writer.Close(), d.db.Close())
}
-// A live-route upsert is an UPDATE followed, only when no row matched, by an
-// INSERT. A live route is keyed by prefix and peer, so an announcement replaces
-// the peer's route for the prefix whatever its origin AS. The UPDATE's
-// rows-affected count (1 for an existing key, 0 for a new one) is what lets the
-// in-memory route counters stay exact without a COUNT(*). Callers hold the
-// database write lock, so no other writer can insert the same key between the
-// two statements. The id column is set only on INSERT, so an updated route
-// keeps its original id, exactly as the previous ON CONFLICT upsert did.
+// A live-route upsert sets the origin AS of the peer's route for the prefix
+// when the announcement changes it, then updates the rest of the route, and
+// inserts the route only when that update matched no row. A live route is keyed
+// by prefix and peer, so an announcement replaces the peer's route for the
+// prefix whatever its origin AS. The update's rows-affected count (1 for an
+// existing key, 0 for a new one) is what lets the in-memory route counters stay
+// exact without a COUNT(*). Callers hold the database write lock, so no other
+// writer can insert the same key between the statements.
+//
+// SQLite rewrites a row's entry in every index on a column an UPDATE sets, even
+// when the value stays the same, so the update sets only the columns that
+// change: mask_length follows from the prefix, and origin_asn has a statement
+// of its own that changes nothing when the origin AS stays the same, as it
+// mostly does.
const (
+ updateLiveRouteOriginV4SQL = `UPDATE live_routes_v4 SET origin_asn = ?
+ WHERE prefix = ? AND peer_ip = ? AND origin_asn != ?`
+
updateLiveRouteV4SQL = `UPDATE live_routes_v4
- SET mask_length = ?, origin_asn = ?, as_path = ?, next_hop = ?,
- last_updated = ? WHERE prefix = ? AND peer_ip = ?`
+ SET as_path = ?, next_hop = ?, last_updated = ?
+ WHERE prefix = ? AND peer_ip = ?`
insertLiveRouteV4SQL = `INSERT INTO live_routes_v4
- (id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
- last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+ (prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
+ last_updated) VALUES (?, ?, ?, ?, ?, ?, ?)`
+
+ updateLiveRouteOriginV6SQL = `UPDATE live_routes_v6 SET origin_asn = ?
+ WHERE prefix = ? AND peer_ip = ? AND origin_asn != ?`
updateLiveRouteV6SQL = `UPDATE live_routes_v6
- SET mask_length = ?, origin_asn = ?, as_path = ?, next_hop = ?,
- last_updated = ? WHERE prefix = ? AND peer_ip = ?`
+ SET as_path = ?, next_hop = ?, last_updated = ?
+ WHERE prefix = ? AND peer_ip = ?`
insertLiveRouteV6SQL = `INSERT INTO live_routes_v6
- (id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
- last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+ (prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
+ last_updated) VALUES (?, ?, ?, ?, ?, ?, ?)`
)
// Before a new route is inserted and after a route is deleted, the write looks
@@ -278,15 +321,22 @@ const (
WHERE prefix = ? AND peer_ip = ?`
)
-// upsertRouteRow updates a live route with the update statement of its IP
-// version's table, inserting it when no row matched. It reports whether a new
-// row was inserted and whether that row is the first live route for its prefix.
+// upsertRouteRow updates a live route with the origin and update statements of
+// its IP version's table, inserting it when no row matched. It reports whether
+// a new row was inserted and whether that row is the first live route for its
+// prefix.
func upsertRouteRow(
- ctx context.Context, upd, ins, has *sql.Stmt, route *LiveRoute,
+ ctx context.Context, setOrigin, upd, ins, has *sql.Stmt, route *LiveRoute,
pathJSON string,
) (bool, bool, error) {
- res, err := upd.ExecContext(ctx, route.MaskLength, route.OriginASN, pathJSON,
- route.NextHop, route.LastUpdated, route.Prefix, route.PeerIP)
+ _, err := setOrigin.ExecContext(ctx, route.OriginASN, route.Prefix,
+ route.PeerIP, route.OriginASN)
+ if err != nil {
+ return false, false, err
+ }
+
+ res, err := upd.ExecContext(ctx, pathJSON, route.NextHop, route.LastUpdated,
+ route.Prefix, route.PeerIP)
if err != nil {
return false, false, err
}
@@ -307,9 +357,9 @@ func upsertRouteRow(
return false, false, err
}
- _, err = ins.ExecContext(ctx, route.ID.String(), route.Prefix,
- route.MaskLength, route.OriginASN, route.PeerIP, pathJSON,
- route.NextHop, route.LastUpdated)
+ _, err = ins.ExecContext(ctx, route.Prefix, route.MaskLength,
+ route.OriginASN, route.PeerIP, pathJSON, route.NextHop,
+ route.LastUpdated)
if err != nil {
return false, false, err
}
@@ -327,13 +377,23 @@ func upsertRoutesInTable(
return 0, nil, nil
}
- family, updateSQL, insertSQL, lookupSQL := ipv4Family, updateLiveRouteV4SQL,
- insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
+ family, originSQL, updateSQL := ipv4Family, updateLiveRouteOriginV4SQL,
+ updateLiveRouteV4SQL
+ insertSQL, lookupSQL := insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
+
if ipVersion == ipVersionV6 {
- family, updateSQL, insertSQL, lookupSQL = ipv6Family,
- updateLiveRouteV6SQL, insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
+ family, originSQL, updateSQL = ipv6Family, updateLiveRouteOriginV6SQL,
+ updateLiveRouteV6SQL
+ insertSQL, lookupSQL = insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
}
+ setOrigin, err := tx.PrepareContext(ctx, originSQL)
+ if err != nil {
+ return 0, nil, fmt.Errorf("failed to prepare %s origin statement: %w",
+ family, err)
+ }
+ defer func() { _ = setOrigin.Close() }()
+
upd, err := tx.PrepareContext(ctx, updateSQL)
if err != nil {
return 0, nil, fmt.Errorf("failed to prepare %s update statement: %w",
@@ -365,8 +425,8 @@ func upsertRoutesInTable(
return 0, nil, fmt.Errorf("failed to encode AS path: %w", err)
}
- inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route,
- string(pathJSON))
+ inserted, newPrefix, err := upsertRouteRow(ctx, setOrigin, upd, ins, has,
+ route, string(pathJSON))
if err != nil {
return 0, nil, fmt.Errorf("failed to upsert route %s: %w",
route.Prefix, err)
@@ -400,9 +460,18 @@ func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error {
}
defer d.rollback(tx)
+ // The rows are written in key order, peer then prefix, so consecutive writes
+ // land on the same index pages. The sort is stable, so of two updates of
+ // one route the later one is still written last.
+ sorted := slices.SortedStableFunc(slices.Values(routes),
+ func(a, b *LiveRoute) int {
+ return cmp.Or(cmp.Compare(a.PeerIP, b.PeerIP),
+ cmp.Compare(a.Prefix, b.Prefix))
+ })
+
var routesV4, routesV6 []*LiveRoute
- for _, route := range routes {
+ for _, route := range sorted {
if route.IPVersion == ipVersionV4 {
routesV4 = append(routesV4, route)
} else {
@@ -546,9 +615,17 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error {
}
defer d.rollback(tx)
+ // The rows are deleted in key order, peer then prefix, as
+ // UpsertLiveRouteBatch writes them.
+ sorted := slices.SortedFunc(slices.Values(deletions),
+ func(a, b LiveRouteDeletion) int {
+ return cmp.Or(cmp.Compare(a.PeerIP, b.PeerIP),
+ cmp.Compare(a.Prefix, b.Prefix))
+ })
+
var deletionsV4, deletionsV6 []LiveRouteDeletion
- for _, del := range deletions {
+ for _, del := range sorted {
if del.IPVersion == ipVersionV4 {
deletionsV4 = append(deletionsV4, del)
} else {
@@ -683,46 +760,11 @@ func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
return int(deletedV4 + deletedV6), nil
}
-// upsertPrefix updates the last_seen time of prefix with the statements of its
-// IP version's table, inserting the prefix when it is new. It reports whether
-// it inserted it.
-func upsertPrefix(
- ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt,
- prefix string, timestamp time.Time,
-) (bool, error) {
- var id string
-
- err := selectStmt.QueryRowContext(ctx, prefix).Scan(&id)
-
- switch {
- case err == nil:
- // Prefix exists, update last_seen
- _, err = updateStmt.ExecContext(ctx, timestamp, prefix)
- if err != nil {
- return false, fmt.Errorf("failed to update prefix %s: %w",
- prefix, err)
- }
-
- return false, nil
- case errors.Is(err, sql.ErrNoRows):
- // Prefix doesn't exist, create it
- _, err = insertStmt.ExecContext(ctx, generateUUID().String(), prefix,
- timestamp, timestamp)
- if err != nil {
- return false, fmt.Errorf("failed to insert prefix %s: %w",
- prefix, err)
- }
-
- return true, nil
- default:
- return false, fmt.Errorf("failed to query prefix %s: %w", prefix, err)
- }
-}
-
-// upsertPrefixesInTable updates the last_seen time of prefixes, all of one IP
-// version, in the prefix table of that version, inserting the ones that are
-// new. It returns how many it inserted.
-func upsertPrefixesInTable(
+// insertPrefixesInTable inserts the prefixes, all of one IP version, that the
+// prefix table of that version does not hold yet, each first seen at its time
+// in prefixes. A prefix already stored is left as it is. It returns how many it
+// inserted.
+func insertPrefixesInTable(
ctx context.Context, tx *loggingTx, prefixes map[string]time.Time,
ipVersion int,
) (int, error) {
@@ -731,33 +773,15 @@ func upsertPrefixesInTable(
}
family := ipv4Family
- selectSQL := "SELECT id FROM prefixes_v4 WHERE prefix = ?"
- updateSQL := "UPDATE prefixes_v4 SET last_seen = ? WHERE prefix = ?"
- insertSQL := "INSERT INTO prefixes_v4 (id, prefix, first_seen, last_seen) " +
- "VALUES (?, ?, ?, ?)"
+ insertSQL := "INSERT INTO prefixes_v4 (prefix, first_seen) VALUES (?, ?) " +
+ "ON CONFLICT (prefix) DO NOTHING"
if ipVersion == ipVersionV6 {
family = ipv6Family
- selectSQL = "SELECT id FROM prefixes_v6 WHERE prefix = ?"
- updateSQL = "UPDATE prefixes_v6 SET last_seen = ? WHERE prefix = ?"
- insertSQL = "INSERT INTO prefixes_v6 (id, prefix, first_seen, last_seen) " +
- "VALUES (?, ?, ?, ?)"
+ insertSQL = "INSERT INTO prefixes_v6 (prefix, first_seen) VALUES (?, ?) " +
+ "ON CONFLICT (prefix) DO NOTHING"
}
- selectStmt, err := tx.PrepareContext(ctx, selectSQL)
- if err != nil {
- return 0, fmt.Errorf("failed to prepare %s select statement: %w",
- family, err)
- }
- defer func() { _ = selectStmt.Close() }()
-
- updateStmt, err := tx.PrepareContext(ctx, updateSQL)
- if err != nil {
- return 0, fmt.Errorf("failed to prepare %s update statement: %w",
- family, err)
- }
- defer func() { _ = updateStmt.Close() }()
-
insertStmt, err := tx.PrepareContext(ctx, insertSQL)
if err != nil {
return 0, fmt.Errorf("failed to prepare %s insert statement: %w",
@@ -767,14 +791,21 @@ func upsertPrefixesInTable(
var newPrefixes int
- for prefix, timestamp := range prefixes {
- inserted, err := upsertPrefix(ctx, selectStmt, updateStmt, insertStmt,
- prefix, timestamp)
+ // The rows are written in key order, as UpsertLiveRouteBatch writes routes.
+ for _, prefix := range slices.Sorted(maps.Keys(prefixes)) {
+ res, err := insertStmt.ExecContext(ctx, prefix, prefixes[prefix])
if err != nil {
- return 0, err
+ return 0, fmt.Errorf("failed to insert prefix %s: %w", prefix, err)
}
- if inserted {
+ // One row for a new prefix, none for one already stored.
+ inserted, err := res.RowsAffected()
+ if err != nil {
+ return 0, fmt.Errorf("failed to count inserted prefix %s: %w",
+ prefix, err)
+ }
+
+ if inserted > 0 {
newPrefixes++
}
}
@@ -782,8 +813,9 @@ func upsertPrefixesInTable(
return newPrefixes, nil
}
-// UpdatePrefixesBatch updates the last_seen time for multiple prefixes in a
-// single transaction, the IPv4 prefixes first and then the IPv6 prefixes.
+// UpdatePrefixesBatch stores, in a single transaction, the prefixes that are
+// not stored yet, each first seen at its time in prefixes: the IPv4 prefixes
+// first and then the IPv6 prefixes.
func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
if len(prefixes) == 0 {
return nil
@@ -811,12 +843,12 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
ctx := context.Background()
- newV4, err := upsertPrefixesInTable(ctx, tx, prefixesV4, ipVersionV4)
+ newV4, err := insertPrefixesInTable(ctx, tx, prefixesV4, ipVersionV4)
if err != nil {
return err
}
- newV6, err := upsertPrefixesInTable(ctx, tx, prefixesV6, ipVersionV6)
+ newV6, err := insertPrefixesInTable(ctx, tx, prefixesV6, ipVersionV6)
if err != nil {
return err
}
@@ -1047,35 +1079,15 @@ func (d *Database) GetOrCreatePrefix(
tableName = "prefixes_v6"
}
- var (
- p Prefix
- idStr string
- )
+ var p Prefix
- query := fmt.Sprintf("SELECT id, prefix, first_seen, last_seen "+
- "FROM %s WHERE prefix = ?", tableName)
+ query := fmt.Sprintf("SELECT prefix, first_seen FROM %s WHERE prefix = ?",
+ tableName)
- err = tx.QueryRow(query, prefix).Scan(&idStr, &p.Prefix, &p.FirstSeen, &p.LastSeen)
+ err = tx.QueryRow(query, prefix).Scan(&p.Prefix, &p.FirstSeen)
if err == nil {
- // Prefix exists, update last_seen
- p.ID, _ = uuid.Parse(idStr)
+ // Prefix exists; the deferred rollback ends the read.
p.IPVersion = ipVersion
- updateQuery := fmt.Sprintf("UPDATE %s SET last_seen = ? WHERE id = ?", tableName)
-
- _, err = tx.Exec(updateQuery, timestamp, p.ID.String())
- if err != nil {
- return nil, err
- }
-
- p.LastSeen = timestamp
-
- err = tx.Commit()
- if err != nil {
- d.logger.Error("Failed to commit transaction for prefix update",
- "prefix", prefix, "error", err)
-
- return nil, err
- }
return &p, nil
}
@@ -1085,12 +1097,11 @@ func (d *Database) GetOrCreatePrefix(
}
// Prefix doesn't exist, create it
- p = Prefix{ID: generateUUID(), Prefix: prefix, IPVersion: ipVersion,
- FirstSeen: timestamp, LastSeen: timestamp}
+ p = Prefix{Prefix: prefix, IPVersion: ipVersion, FirstSeen: timestamp}
insertQuery := fmt.Sprintf("INSERT INTO %s "+
- "(id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)", tableName)
+ "(prefix, first_seen) VALUES (?, ?)", tableName)
- _, err = tx.Exec(insertQuery, p.ID.String(), p.Prefix, p.FirstSeen, p.LastSeen)
+ _, err = tx.Exec(insertQuery, p.Prefix, p.FirstSeen)
if err != nil {
return nil, err
}
@@ -1389,37 +1400,44 @@ func (d *Database) UpsertLiveRoute(route *LiveRoute) error {
return fmt.Errorf("failed to encode AS path: %w", err)
}
- updateSQL, insertSQL, lookupSQL := updateLiveRouteV4SQL,
- insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
+ originSQL, updateSQL := updateLiveRouteOriginV4SQL, updateLiveRouteV4SQL
+ insertSQL, lookupSQL := insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
+
if route.IPVersion == ipVersionV6 {
- updateSQL, insertSQL, lookupSQL = updateLiveRouteV6SQL,
- insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
+ originSQL, updateSQL = updateLiveRouteOriginV6SQL, updateLiveRouteV6SQL
+ insertSQL, lookupSQL = insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
}
ctx := context.Background()
// The write lock is held, so no other writer can insert this key between the
// update and the insert even though they are separate autocommit statements.
- upd, err := d.db.PrepareContext(ctx, updateSQL)
+ setOrigin, err := d.writer.PrepareContext(ctx, originSQL)
+ if err != nil {
+ return fmt.Errorf("failed to prepare origin statement: %w", err)
+ }
+ defer func() { _ = setOrigin.Close() }()
+
+ upd, err := d.writer.PrepareContext(ctx, updateSQL)
if err != nil {
return fmt.Errorf("failed to prepare update statement: %w", err)
}
defer func() { _ = upd.Close() }()
- ins, err := d.db.PrepareContext(ctx, insertSQL)
+ ins, err := d.writer.PrepareContext(ctx, insertSQL)
if err != nil {
return fmt.Errorf("failed to prepare insert statement: %w", err)
}
defer func() { _ = ins.Close() }()
- has, err := d.db.PrepareContext(ctx, lookupSQL)
+ has, err := d.writer.PrepareContext(ctx, lookupSQL)
if err != nil {
return fmt.Errorf("failed to prepare prefix lookup statement: %w", err)
}
defer func() { _ = has.Close() }()
- inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route,
- string(pathJSON))
+ inserted, newPrefix, err := upsertRouteRow(ctx, setOrigin, upd, ins, has,
+ route, string(pathJSON))
if err != nil {
return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err)
}
@@ -2008,7 +2026,7 @@ func (d *Database) UpdateASNWHOIS(
WHERE asn = ?
`
- _, err := d.db.ExecContext(ctx, query,
+ _, err := d.writer.ExecContext(ctx, query,
update.ASName,
update.OrgName,
update.OrgID,
@@ -2165,7 +2183,8 @@ func (d *Database) GetIPInfoContext(
// Vacuum runs incremental vacuum to return up to 1000 unused pages to the
// filesystem. Like Analyze and Checkpoint, it holds the database lock, so it
// and the batch writes wait for each other instead of failing on SQLite's
-// write lock.
+// write lock, and returns ctx's error without running if ctx ended while it
+// waited: the writing connection it uses does not check ctx itself.
func (d *Database) Vacuum(ctx context.Context) error {
// Free up to 1000 pages per call (~4MB with default 4KB page size)
// This keeps each vacuum operation quick
@@ -2174,7 +2193,12 @@ func (d *Database) Vacuum(ctx context.Context) error {
d.lock("Vacuum")
defer d.unlock()
- rows, err := d.db.QueryContext(ctx,
+ err := ctx.Err()
+ if err != nil {
+ return fmt.Errorf("failed to run incremental vacuum: %w", err)
+ }
+
+ rows, err := d.writer.QueryContext(ctx,
fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree))
if err != nil {
return fmt.Errorf("failed to run incremental vacuum: %w", err)
@@ -2204,9 +2228,14 @@ func (d *Database) Analyze(ctx context.Context) error {
d.lock("Analyze")
defer d.unlock()
+ err := ctx.Err()
+ if err != nil {
+ return fmt.Errorf("failed to analyze database: %w", err)
+ }
+
// analysis_limit is a setting of the connection, so it is set in the same
// call as PRAGMA optimize, which runs both statements on one connection.
- _, err := d.db.ExecContext(ctx, fmt.Sprintf(
+ _, err = d.writer.ExecContext(ctx, fmt.Sprintf(
"PRAGMA analysis_limit=%d; PRAGMA optimize=%#x",
analysisLimit, optimizeAllTables))
if err != nil {
@@ -2223,7 +2252,12 @@ func (d *Database) Checkpoint(ctx context.Context) error {
d.lock("Checkpoint")
defer d.unlock()
- _, err := d.db.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
+ err := ctx.Err()
+ if err != nil {
+ return fmt.Errorf("failed to checkpoint WAL: %w", err)
+ }
+
+ _, err = d.writer.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
if err != nil {
return fmt.Errorf("failed to checkpoint WAL: %w", err)
}
@@ -2279,9 +2313,9 @@ func (d *Database) unlock() {
"held_by", lockedBy, "duration_ms", holdDuration.Milliseconds())
}
-// beginTx starts a new transaction with logging
+// beginTx starts a new transaction on the writing connection, with logging
func (d *Database) beginTx() (*loggingTx, error) {
- tx, err := d.db.BeginTx(context.Background(), nil)
+ tx, err := d.writer.BeginTx(context.Background(), nil)
if err != nil {
return nil, err
}
diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go
index d93b188..92768c9 100644
--- a/internal/database/database_internal_test.go
+++ b/internal/database/database_internal_test.go
@@ -6,15 +6,16 @@ import (
"errors"
"fmt"
"log/slog"
+ "math/rand/v2"
"net/netip"
"slices"
+ "strconv"
"sync"
"testing"
"time"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/logger"
- "github.com/google/uuid"
"github.com/mattn/go-sqlite3"
)
@@ -33,6 +34,9 @@ const vacuumTestRoutes = 2000
// once so each is a distinct SQLite connection that parsed the DSN.
const heldConnections = 5
+// bytesPerKiB turns a cache_size in KiB into bytes.
+const bytesPerKiB = 1024
+
// testPeerIP is the peer every route in the IP lookup test is learned from.
const testPeerIP = "192.0.2.254"
@@ -293,7 +297,7 @@ func TestLiveRouteTablesHoldOneRoutePerPrefixAndPeer(t *testing.T) {
t.Fatalf("UpsertLiveRoute(%s): %v", tt.route.Prefix, err)
}
- _, err = db.db.ExecContext(t.Context(), tt.insertSQL, uuid.NewString(),
+ _, err = db.db.ExecContext(t.Context(), tt.insertSQL,
tt.route.Prefix, tt.route.MaskLength, otherOriginASN,
tt.route.PeerIP, "[]", tt.route.NextHop, ts)
@@ -317,7 +321,6 @@ func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) {
for prefix, asn := range origins {
routes = append(routes, &LiveRoute{
- ID: uuid.New(),
Prefix: prefix,
MaskLength: netip.MustParsePrefix(prefix).Bits(),
IPVersion: detectIPVersion(prefix),
@@ -337,7 +340,7 @@ func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) {
// TestConnectionPoolPragmas holds several pooled connections open at once and
// checks each one carries the per-connection settings from the DSN, plus the
-// process-wide hard heap limit, and sees the new file with auto_vacuum
+// process-wide heap limits, and sees the new file with auto_vacuum
// incremental.
func TestConnectionPoolPragmas(t *testing.T) {
t.Parallel()
@@ -372,9 +375,9 @@ func TestConnectionPoolPragmas(t *testing.T) {
for i, c := range conns {
cacheSize := readPragma(ctx, t, c, i, "cache_size")
- if cacheSize != sqliteCacheSizeKiB {
+ if cacheSize != sqliteReadCacheSizeKiB {
t.Errorf("conn %d: cache_size = %d, want %d",
- i, cacheSize, sqliteCacheSizeKiB)
+ i, cacheSize, sqliteReadCacheSizeKiB)
}
busyTimeout := readPragma(ctx, t, c, i, "busy_timeout")
@@ -389,6 +392,12 @@ func TestConnectionPoolPragmas(t *testing.T) {
i, tempStore, tempStoreMemory)
}
+ softHeapLimit := readPragma(ctx, t, c, i, "soft_heap_limit")
+ if softHeapLimit != sqliteSoftHeapLimitBytes {
+ t.Errorf("conn %d: soft_heap_limit = %d, want %d",
+ i, softHeapLimit, sqliteSoftHeapLimitBytes)
+ }
+
hardHeapLimit := readPragma(ctx, t, c, i, "hard_heap_limit")
if hardHeapLimit != sqliteHardHeapLimitBytes {
t.Errorf("conn %d: hard_heap_limit = %d, want %d",
@@ -403,6 +412,51 @@ func TestConnectionPoolPragmas(t *testing.T) {
}
}
+// TestWritingConnectionCache checks that the connection kept for writing has
+// the large cache, and that the soft heap limit is above the caches of all the
+// connections together: SQLite shrinks the caches to stay under it.
+func TestWritingConnectionCache(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ ctx := t.Context()
+
+ var writeCacheSize, readCacheSize, softHeapLimit int64
+
+ err = db.writer.QueryRowContext(ctx, "PRAGMA cache_size").
+ Scan(&writeCacheSize)
+ if err != nil {
+ t.Fatalf("failed to read the writing connection's cache_size: %v", err)
+ }
+
+ if writeCacheSize != sqliteWriteCacheSizeKiB {
+ t.Errorf("writing connection: cache_size = %d, want %d",
+ writeCacheSize, sqliteWriteCacheSizeKiB)
+ }
+
+ err = db.db.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&readCacheSize)
+ if err != nil {
+ t.Fatalf("failed to read a reading connection's cache_size: %v", err)
+ }
+
+ err = db.db.QueryRowContext(ctx, "PRAGMA soft_heap_limit").
+ Scan(&softHeapLimit)
+ if err != nil {
+ t.Fatalf("failed to read soft_heap_limit: %v", err)
+ }
+
+ cachesBytes := -bytesPerKiB * (writeCacheSize + (maxConns-1)*readCacheSize)
+ if softHeapLimit <= cachesBytes {
+ t.Errorf("soft_heap_limit = %d, not above the %d bytes of the caches",
+ softHeapLimit, cachesBytes)
+ }
+}
+
// readPragma reads the integer value of PRAGMA name on connection i, c.
func readPragma(
ctx context.Context, t *testing.T, c *sql.Conn, i int, name string,
@@ -1090,3 +1144,380 @@ func TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes(t *testing.T) {
routes, kept.Prefix)
}
}
+
+// TestLiveRouteUpdateSetsOriginWhenItChanges announces a peer's route for a
+// prefix, then again with the same origin AS and a new path, then with a new
+// origin AS. Each time the route must carry the announced origin AS, path and
+// time, and the AS details must list the prefix under that origin AS only. The
+// update sets the origin AS with a statement of its own, which changes nothing
+// while the origin AS stays the same.
+func TestLiveRouteUpdateSetsOriginWhenItChanges(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ const firstOrigin, secondOrigin = 64500, 64501
+
+ ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+ for _, asn := range []int{firstOrigin, secondOrigin} {
+ _, err := db.GetOrCreateASN(asn, ts)
+ if err != nil {
+ t.Fatalf("GetOrCreateASN(%d): %v", asn, err)
+ }
+ }
+
+ route := mkV4Route(t, "198.51.100.0/24", firstOrigin, ts)
+
+ for _, announced := range []struct {
+ origin int
+ path []int
+ }{
+ {firstOrigin, []int{firstOrigin}},
+ {firstOrigin, []int{secondOrigin, firstOrigin}},
+ {secondOrigin, []int{firstOrigin, secondOrigin}},
+ } {
+ ts = ts.Add(time.Minute)
+ route.OriginASN, route.ASPath, route.LastUpdated =
+ announced.origin, announced.path, ts
+
+ err := db.UpsertLiveRouteBatch([]*LiveRoute{route})
+ if err != nil {
+ t.Fatalf("UpsertLiveRouteBatch: %v", err)
+ }
+
+ routes, _, err := db.GetPrefixDetails(route.Prefix)
+ if err != nil {
+ t.Fatalf("GetPrefixDetails: %v", err)
+ }
+
+ if len(routes) != 1 || routes[0].OriginASN != announced.origin ||
+ !slices.Equal(routes[0].ASPath, announced.path) ||
+ !routes[0].LastUpdated.Equal(ts) {
+ t.Errorf("after announcing origin AS%d, path %v at %s: routes %+v",
+ announced.origin, announced.path, ts, routes)
+ }
+
+ for _, asn := range []int{firstOrigin, secondOrigin} {
+ _, prefixes, err := db.GetASDetails(asn)
+ if err != nil {
+ t.Fatalf("GetASDetails(%d): %v", asn, err)
+ }
+
+ listed := len(prefixes) == 1 && prefixes[0].Prefix == route.Prefix
+ if listed != (asn == announced.origin) {
+ t.Errorf("after announcing origin AS%d, AS%d lists %+v",
+ announced.origin, asn, prefixes)
+ }
+ }
+ }
+}
+
+// TestUpdatePrefixesBatchKeepsFirstSeen writes a prefix, then a batch with it
+// and a new prefix. Only the new prefix is inserted and counted, and the known
+// one keeps its first_seen time.
+func TestUpdatePrefixesBatchKeepsFirstSeen(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ const known = "198.51.100.0/24"
+
+ first := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+ later := first.Add(time.Hour)
+
+ err = db.UpdatePrefixesBatch(map[string]time.Time{known: first})
+ if err != nil {
+ t.Fatalf("UpdatePrefixesBatch: %v", err)
+ }
+
+ err = db.UpdatePrefixesBatch(map[string]time.Time{
+ known: later, testV6Prefix: later,
+ })
+ if err != nil {
+ t.Fatalf("UpdatePrefixesBatch: %v", err)
+ }
+
+ stats, err := db.GetStats()
+ if err != nil {
+ t.Fatalf("GetStats: %v", err)
+ }
+
+ if stats.IPv4Prefixes != 1 || stats.IPv6Prefixes != 1 {
+ t.Errorf("prefix counts = (v4 %d, v6 %d), want (1, 1)",
+ stats.IPv4Prefixes, stats.IPv6Prefixes)
+ }
+
+ for _, tt := range []struct {
+ query, prefix string
+ firstSeen time.Time
+ }{
+ {
+ query: "SELECT first_seen FROM prefixes_v4 WHERE prefix = ?",
+ prefix: known, firstSeen: first,
+ },
+ {
+ query: "SELECT first_seen FROM prefixes_v6 WHERE prefix = ?",
+ prefix: testV6Prefix, firstSeen: later,
+ },
+ } {
+ var firstSeen time.Time
+
+ err := db.db.QueryRowContext(t.Context(), tt.query, tt.prefix).
+ Scan(&firstSeen)
+ if err != nil {
+ t.Fatalf("reading %s: %v", tt.prefix, err)
+ }
+
+ if !firstSeen.Equal(tt.firstSeen) {
+ t.Errorf("%s: first seen %s, want %s", tt.prefix, firstSeen,
+ tt.firstSeen)
+ }
+ }
+}
+
+// TestUpsertLiveRouteBatchWritesInKeyOrder writes new routes of two peers out
+// of key order, the last one a second announcement of the first, and checks
+// that the rows were inserted in key order, peer then prefix (each new row's id
+// is one above the last), and that the route announced twice carries the later
+// announcement.
+func TestUpsertLiveRouteBatchWritesInKeyOrder(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ const (
+ firstOrigin, laterOrigin = 64500, 64501
+ // Three prefixes, in text order.
+ firstPrefix = "192.0.2.0/24"
+ secondPrefix = "198.51.100.0/24"
+ thirdPrefix = "203.0.113.0/24"
+ )
+
+ type row struct {
+ peer, prefix string
+ origin int
+ }
+
+ ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+ announcements := []row{
+ {secondV4PeerIP, secondPrefix, firstOrigin},
+ {testV4PeerIP, thirdPrefix, firstOrigin},
+ {secondV4PeerIP, firstPrefix, firstOrigin},
+ {testV4PeerIP, secondPrefix, firstOrigin},
+ {secondV4PeerIP, secondPrefix, laterOrigin},
+ }
+
+ routes := make([]*LiveRoute, 0, len(announcements))
+
+ for _, announced := range announcements {
+ route := mkV4Route(t, announced.prefix, announced.origin, ts)
+ route.PeerIP = announced.peer
+ routes = append(routes, route)
+ }
+
+ err = db.UpsertLiveRouteBatch(routes)
+ if err != nil {
+ t.Fatalf("UpsertLiveRouteBatch: %v", err)
+ }
+
+ rows, err := db.db.QueryContext(t.Context(),
+ "SELECT peer_ip, prefix, origin_asn FROM live_routes_v4 ORDER BY id")
+ if err != nil {
+ t.Fatalf("reading the routes: %v", err)
+ }
+ defer func() { _ = rows.Close() }()
+
+ var stored []row
+
+ for rows.Next() {
+ var r row
+
+ err := rows.Scan(&r.peer, &r.prefix, &r.origin)
+ if err != nil {
+ t.Fatalf("reading a route: %v", err)
+ }
+
+ stored = append(stored, r)
+ }
+
+ err = rows.Err()
+ if err != nil {
+ t.Fatalf("reading the routes: %v", err)
+ }
+
+ want := []row{
+ {testV4PeerIP, secondPrefix, firstOrigin},
+ {testV4PeerIP, thirdPrefix, firstOrigin},
+ {secondV4PeerIP, firstPrefix, firstOrigin},
+ {secondV4PeerIP, secondPrefix, laterOrigin},
+ }
+
+ if !slices.Equal(stored, want) {
+ t.Errorf("routes in insertion order = %+v, want %+v", stored, want)
+ }
+}
+
+// Parameters for BenchmarkPrefixBatchOnFiveMillionRoutes.
+const (
+ // benchPeers and benchPrefixes size the table the batches are written to:
+ // every peer has a route for every prefix, 5 million routes in all.
+ benchPeers = 10
+ benchPrefixes = 500000
+ // benchBatchSize is how many updates each batch holds, as many as a full
+ // prefix handler batch (prefixBatchSize in internal/routewatch).
+ benchBatchSize = 25000
+ // benchMaxBatchSize is the largest batch the prefix handler writes, the
+ // one it holds while a slow write runs (prefixBatchMaxSize in
+ // internal/routewatch).
+ benchMaxBatchSize = 250000
+ // benchWithdrawalEvery makes every fifth update of a timed batch a
+ // withdrawal; the others are announcements.
+ benchWithdrawalEvery = 5
+ // benchOrigins is how many origin ASes the prefixes are spread over.
+ benchOrigins = 50000
+ // benchFirstASN is the lowest AS number in the benchmark's AS paths.
+ benchFirstASN = 64500
+ // benchMaskLength is the mask length of every prefix in the benchmark.
+ benchMaskLength = 24
+)
+
+// BenchmarkPrefixBatchOnFiveMillionRoutes times the writes the prefix handler
+// makes for one batch of announcements and withdrawals of routes picked at
+// random, to a database that already holds 5 million live routes: 500,000
+// IPv4 prefixes, each from 10 peers. It times a full batch of 25,000 updates,
+// then the largest batch, 250,000. It builds that database first, one batch at
+// a time, which takes some minutes and a few GB of disk under TMPDIR. It is
+// run by hand, not by make test, for example in the test image:
+//
+// go test -run '^$' -bench FiveMillion -benchtime 10x ./internal/database/
+func BenchmarkPrefixBatchOnFiveMillionRoutes(b *testing.B) {
+ db, err := New(&config.Config{StateDir: b.TempDir()}, logger.New())
+ if err != nil {
+ b.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ //nolint:gosec // The benchmark needs the same batches every run.
+ rng := rand.New(rand.NewPCG(1, 2))
+ ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+ // Each peer sends its whole table, as when its session comes up.
+ for peer := range benchPeers {
+ for first := 0; first < benchPrefixes; first += benchBatchSize {
+ routes := make([]*LiveRoute, 0, benchBatchSize)
+ for prefix := first; prefix < first+benchBatchSize; prefix++ {
+ routes = append(routes, benchRoute(rng, prefix, peer, ts))
+ }
+
+ writeBenchBatch(b, db, routes, nil, ts)
+ }
+ }
+
+ for _, batchSize := range []int{benchBatchSize, benchMaxBatchSize} {
+ b.Run(strconv.Itoa(batchSize), func(b *testing.B) {
+ for b.Loop() {
+ ts = ts.Add(time.Second)
+
+ var (
+ routes []*LiveRoute
+ withdrawals []LiveRouteDeletion
+ )
+
+ for i := range batchSize {
+ route := benchRoute(rng, rng.IntN(benchPrefixes),
+ rng.IntN(benchPeers), ts)
+ if i%benchWithdrawalEvery == 0 {
+ withdrawals = append(withdrawals, LiveRouteDeletion{
+ Prefix: route.Prefix, PeerIP: route.PeerIP,
+ IPVersion: ipVersionV4,
+ })
+ } else {
+ routes = append(routes, route)
+ }
+ }
+
+ writeBenchBatch(b, db, routes, withdrawals, ts)
+ }
+ })
+ }
+}
+
+// benchRoute returns the route peer has for the prefix numbered prefix, a /24
+// from 1.0.0.0/24 on, with a new AS path of three to six ASes from the peer's
+// AS to the prefix's origin AS.
+func benchRoute(rng *rand.Rand, prefix, peer int, ts time.Time) *LiveRoute {
+ const minPathLength, extraPathLength = 3, 4
+
+ origin := benchFirstASN + prefix%benchOrigins
+ pathLength := minPathLength + rng.IntN(extraPathLength)
+
+ path := make([]int, 0, pathLength)
+ path = append(path, benchFirstASN+peer)
+
+ for len(path) < pathLength-1 {
+ path = append(path, benchFirstASN+rng.IntN(benchOrigins))
+ }
+
+ path = append(path, origin)
+ peerIP := fmt.Sprintf("198.51.100.%d", 1+peer)
+
+ return &LiveRoute{
+ Prefix: fmt.Sprintf("%d.%d.%d.0/%d",
+ 1+prefix>>16, (prefix>>8)%256, prefix%256, benchMaskLength),
+ MaskLength: benchMaskLength,
+ IPVersion: ipVersionV4,
+ OriginASN: origin,
+ PeerIP: peerIP,
+ ASPath: path,
+ NextHop: peerIP,
+ LastUpdated: ts,
+ }
+}
+
+// writeBenchBatch writes what the prefix handler writes for a batch: the
+// announced routes, the withdrawals, and every prefix the batch names, of
+// which only the ones not stored yet are inserted.
+func writeBenchBatch(
+ b *testing.B, db *Database, routes []*LiveRoute,
+ withdrawals []LiveRouteDeletion, ts time.Time,
+) {
+ b.Helper()
+
+ prefixes := make(map[string]time.Time, len(routes)+len(withdrawals))
+ for _, route := range routes {
+ prefixes[route.Prefix] = ts
+ }
+
+ for _, withdrawal := range withdrawals {
+ prefixes[withdrawal.Prefix] = ts
+ }
+
+ err := db.UpsertLiveRouteBatch(routes)
+ if err != nil {
+ b.Fatalf("UpsertLiveRouteBatch: %v", err)
+ }
+
+ err = db.DeleteLiveRouteBatch(withdrawals)
+ if err != nil {
+ b.Fatalf("DeleteLiveRouteBatch: %v", err)
+ }
+
+ err = db.UpdatePrefixesBatch(prefixes)
+ if err != nil {
+ b.Fatalf("UpdatePrefixesBatch: %v", err)
+ }
+}
diff --git a/internal/database/models.go b/internal/database/models.go
index 3872f95..5392a4c 100644
--- a/internal/database/models.go
+++ b/internal/database/models.go
@@ -35,13 +35,11 @@ type ASN struct {
}
// Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6)
-// and first/last seen timestamps.
+// and the time it was first seen.
type Prefix struct {
- ID uuid.UUID
Prefix string
IPVersion int // 4 or 6
FirstSeen time.Time
- LastSeen time.Time
}
// Announcement represents a BGP announcement or withdrawal event,
@@ -71,7 +69,6 @@ type ASNPeering struct {
//
//nolint:tagliatelle // field names are the existing JSON API
type LiveRoute struct {
- ID uuid.UUID `json:"id"`
Prefix string `json:"prefix"`
MaskLength int `json:"mask_length"`
IPVersion int `json:"ip_version"`
diff --git a/internal/database/schema.sql b/internal/database/schema.sql
index ef2f5dd..6bd65a6 100644
--- a/internal/database/schema.sql
+++ b/internal/database/schema.sql
@@ -27,21 +27,18 @@ CREATE TABLE IF NOT EXISTS asns (
whois_updated_at DATETIME -- when we last fetched WHOIS data
);
--- IPv4 prefixes table
+-- IPv4 prefixes table. Every read and write finds a row by its prefix, so the
+-- rows are stored by prefix (WITHOUT ROWID) and need no index of their own.
CREATE TABLE IF NOT EXISTS prefixes_v4 (
- id TEXT PRIMARY KEY,
- prefix TEXT UNIQUE NOT NULL,
- first_seen DATETIME NOT NULL,
- last_seen DATETIME NOT NULL
-);
+ prefix TEXT PRIMARY KEY,
+ first_seen DATETIME NOT NULL
+) WITHOUT ROWID;
--- IPv6 prefixes table
+-- IPv6 prefixes table, stored as prefixes_v4 is.
CREATE TABLE IF NOT EXISTS prefixes_v6 (
- id TEXT PRIMARY KEY,
- prefix TEXT UNIQUE NOT NULL,
- first_seen DATETIME NOT NULL,
- last_seen DATETIME NOT NULL
-);
+ prefix TEXT PRIMARY KEY,
+ first_seen DATETIME NOT NULL
+) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS announcements (
id TEXT PRIMARY KEY,
@@ -75,11 +72,6 @@ CREATE TABLE IF NOT EXISTS bgp_peers (
last_message_type TEXT
);
--- Indexes for prefixes_v4 table
-CREATE INDEX IF NOT EXISTS idx_prefixes_v4_prefix ON prefixes_v4(prefix);
-
--- Indexes for prefixes_v6 table
-CREATE INDEX IF NOT EXISTS idx_prefixes_v6_prefix ON prefixes_v6(prefix);
CREATE INDEX IF NOT EXISTS idx_announcements_timestamp ON announcements(timestamp);
CREATE INDEX IF NOT EXISTS idx_announcements_prefix_id ON announcements(prefix_id);
CREATE INDEX IF NOT EXISTS idx_announcements_peer_asn ON announcements(peer_asn);
@@ -97,9 +89,14 @@ CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
--- IPv4 routing table maintained by PrefixHandler
+-- IPv4 routing table maintained by PrefixHandler. A route write changes the
+-- table and its entry in every index on a column the write sets, so the table
+-- has only the indexes that the queries named with them use.
CREATE TABLE IF NOT EXISTS live_routes_v4 (
- id TEXT PRIMARY KEY,
+ -- The row id, which the prefix-length pages' sample draws through the
+ -- mask_length index. Declared as a column, it stays the same for as long
+ -- as the route is live, also through a VACUUM.
+ id INTEGER PRIMARY KEY,
prefix TEXT NOT NULL,
mask_length INTEGER NOT NULL, -- CIDR mask length (0-32)
origin_asn INTEGER NOT NULL,
@@ -107,15 +104,15 @@ CREATE TABLE IF NOT EXISTS live_routes_v4 (
as_path TEXT NOT NULL, -- JSON array
next_hop TEXT NOT NULL,
last_updated DATETIME NOT NULL,
- -- A peer has at most one route for a prefix. peer_ip comes first so that
- -- this key also finds all of a peer's routes, when its BGP session goes
- -- down; lookups by prefix use the prefix index below.
+ -- A peer has at most one route for a prefix. This key finds the route a
+ -- write updates or deletes and, peer_ip coming first, all of a peer's
+ -- routes when its BGP session goes down.
UNIQUE(peer_ip, prefix)
);
--- IPv6 routing table maintained by PrefixHandler
+-- IPv6 routing table maintained by PrefixHandler, as live_routes_v4.
CREATE TABLE IF NOT EXISTS live_routes_v6 (
- id TEXT PRIMARY KEY,
+ id INTEGER PRIMARY KEY,
prefix TEXT NOT NULL,
mask_length INTEGER NOT NULL, -- CIDR mask length (0-128)
origin_asn INTEGER NOT NULL,
@@ -123,24 +120,29 @@ CREATE TABLE IF NOT EXISTS live_routes_v6 (
as_path TEXT NOT NULL, -- JSON array
next_hop TEXT NOT NULL,
last_updated DATETIME NOT NULL,
- -- As in live_routes_v4: peer_ip first, to find all of a peer's routes.
UNIQUE(peer_ip, prefix)
);
--- Indexes for live_routes_v4 table
--- Finds a prefix's newest route in one search, whatever the number of peers.
+-- Indexes for both live route tables, and the queries that use each.
+--
+-- prefix, last_updated: a prefix's routes, for the prefix page and its API
+-- answer, the IP lookup and its peer count, and the check whether a prefix
+-- still has a live route; and a prefix's newest route for the prefix-length
+-- pages, in one search whatever the number of peers.
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
--- Index to optimize prefix distribution queries
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
-
--- Indexes for live_routes_v6 table
--- Finds a prefix's newest route in one search, whatever the number of peers.
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
+-- mask_length: the prefix-length pages' random sample, the row id range of a
+-- mask length and the first route of it at or after a row id.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask_length);
+-- mask_length, prefix: the prefix-length pages' list of a mask length's
+-- prefixes in text order, and the prefix distribution read at startup.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
+CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
+-- origin_asn: the prefixes an AS originates, for the AS page and its API answer.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_asn ON live_routes_v6(origin_asn);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
--- Index to optimize prefix distribution queries
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
\ No newline at end of file
+-- last_updated: the oldest and newest route times in /api/v1/stats and
+-- /status.json.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
+CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
\ No newline at end of file
diff --git a/internal/database/slowquery.go b/internal/database/slowquery.go
index d027211..13a0ac8 100644
--- a/internal/database/slowquery.go
+++ b/internal/database/slowquery.go
@@ -18,12 +18,12 @@ func logSlowQuery(logger *logger.Logger, query string, start time.Time) {
}
}
-// exec wraps Exec with slow query logging
+// exec runs a statement on the writing connection, with slow query logging
func (d *Database) exec(query string, args ...any) error {
start := time.Now()
defer logSlowQuery(d.logger, query, start)
- _, err := d.db.ExecContext(context.Background(), query, args...)
+ _, err := d.writer.ExecContext(context.Background(), query, args...)
return err
}
diff --git a/internal/routewatch/app_integration_internal_test.go b/internal/routewatch/app_integration_internal_test.go
index bdab0a4..8f094ca 100644
--- a/internal/routewatch/app_integration_internal_test.go
+++ b/internal/routewatch/app_integration_internal_test.go
@@ -18,7 +18,6 @@ import (
"git.eeqj.de/sneak/routewatch/internal/ristypes"
"git.eeqj.de/sneak/routewatch/internal/server"
"git.eeqj.de/sneak/routewatch/internal/streamer"
- "github.com/google/uuid"
)
// mockStore is a mock implementation of database.Store for testing
@@ -83,9 +82,7 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
defer m.mu.Unlock()
for prefix, timestamp := range prefixes {
- if p, exists := m.Prefixes[prefix]; exists {
- p.LastSeen = timestamp
- } else {
+ if _, exists := m.Prefixes[prefix]; !exists {
const (
ipVersionV4 = 4
ipVersionV6 = 6
@@ -97,11 +94,9 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
}
m.Prefixes[prefix] = &database.Prefix{
- ID: uuid.New(),
Prefix: prefix,
IPVersion: ipVersion,
FirstSeen: timestamp,
- LastSeen: timestamp,
}
}
}
@@ -117,8 +112,6 @@ func (m *mockStore) GetOrCreatePrefix(
defer m.mu.Unlock()
if p, exists := m.Prefixes[prefix]; exists {
- p.LastSeen = timestamp
-
return p, nil
}
@@ -133,11 +126,9 @@ func (m *mockStore) GetOrCreatePrefix(
}
p := &database.Prefix{
- ID: uuid.New(),
Prefix: prefix,
IPVersion: ipVersion,
FirstSeen: timestamp,
- LastSeen: timestamp,
}
m.Prefixes[prefix] = p
m.PrefixCount++
@@ -370,11 +361,9 @@ func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
// Track prefix
if _, exists := m.Prefixes[route.Prefix]; !exists {
m.Prefixes[route.Prefix] = &database.Prefix{
- ID: uuid.New(),
Prefix: route.Prefix,
IPVersion: route.IPVersion,
FirstSeen: route.LastUpdated,
- LastSeen: route.LastUpdated,
}
m.PrefixCount++
diff --git a/internal/routewatch/prefixhandler.go b/internal/routewatch/prefixhandler.go
index 4858e76..c3ae80e 100644
--- a/internal/routewatch/prefixhandler.go
+++ b/internal/routewatch/prefixhandler.go
@@ -12,7 +12,6 @@ import (
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/metrics"
"git.eeqj.de/sneak/routewatch/internal/ristypes"
- "github.com/google/uuid"
)
const (
@@ -461,7 +460,6 @@ func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute
// Create live route record
liveRoute := &database.LiveRoute{
- ID: uuid.New(),
Prefix: update.prefix,
MaskLength: maskLength,
IPVersion: ipVersion,
diff --git a/internal/server/handlers_internal_test.go b/internal/server/handlers_internal_test.go
index ff68f08..8ea16a3 100644
--- a/internal/server/handlers_internal_test.go
+++ b/internal/server/handlers_internal_test.go
@@ -19,7 +19,6 @@ import (
"git.eeqj.de/sneak/routewatch/internal/metrics"
"git.eeqj.de/sneak/routewatch/internal/streamer"
"git.eeqj.de/sneak/routewatch/pkg/asinfo"
- "github.com/google/uuid"
)
// The prefix and peer addresses the prefix page tests store routes for.
@@ -114,12 +113,12 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
err = db.UpsertLiveRouteBatch([]*database.LiveRoute{
{
- ID: uuid.New(), Prefix: "198.51.100.0/24", MaskLength: 24, IPVersion: 4,
+ Prefix: "198.51.100.0/24", MaskLength: 24, IPVersion: 4,
OriginASN: 64500, PeerIP: "192.0.2.1", ASPath: []int{64500},
NextHop: "192.0.2.1", LastUpdated: ts,
},
{
- ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6,
+ Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6,
OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501},
NextHop: testPeerV6, LastUpdated: ts,
},
@@ -204,12 +203,12 @@ func TestPrefixDetailHandlersParsePrefix(t *testing.T) {
err = db.UpsertLiveRouteBatch([]*database.LiveRoute{
{
- ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
+ Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
OriginASN: 64500, PeerIP: testPeerV4, ASPath: []int{64500},
NextHop: testPeerV4, LastUpdated: ts,
},
{
- ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6,
+ Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6,
OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501},
NextHop: testPeerV6, LastUpdated: ts,
},
@@ -287,12 +286,12 @@ func TestPrefixDetailShowsOriginNames(t *testing.T) {
err = db.UpsertLiveRouteBatch([]*database.LiveRoute{
{
- ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
+ Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
OriginASN: origins[0], PeerIP: testPeerV4,
ASPath: []int{origins[0]}, NextHop: testPeerV4, LastUpdated: ts,
},
{
- ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
+ Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
OriginASN: origins[1], PeerIP: "198.51.100.2",
ASPath: []int{origins[1]}, NextHop: "198.51.100.2", LastUpdated: ts,
},
@@ -347,7 +346,7 @@ func TestPagesSayOriginZeroIsNoKnownOrigin(t *testing.T) {
t.Cleanup(func() { _ = db.Close() })
err = db.UpsertLiveRouteBatch([]*database.LiveRoute{{
- ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
+ Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
OriginASN: 0, PeerIP: testPeerV4, ASPath: []int{64500, 64501, 64502},
NextHop: testPeerV4, LastUpdated: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
}})
--
2.54.0
From ef32082339da22ffd2f770f00f7903fbd1a4102a Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Mon, 19 Oct 2026 16:09:22 +0200
Subject: [PATCH 21/45] Bring README and TODO.md up to date with the code
(closes #72)
README "HTTP Endpoints" now lists every route the server registers with
what it answers, errors included: / is a home page and /ip/{addr} answers
JSON, neither redirects; /ip/ looks up the caller; the IP endpoints take
host names and comma-separated lists; 400, 404 and 504 are described, and
the health check's envelope. "Database Schema" matches schema.sql, and
other sentences the code contradicted are corrected. TODO.md gets a dated
line for each issue landed since 2026-10-03, with Status and Next Step
current.
Model: opus-5-5
---
README.md | 112 +++++++++++++++++++++++++++++++++++++++---------------
TODO.md | 89 ++++++++++++++++++++++++++++++++++++-------
2 files changed, 157 insertions(+), 44 deletions(-)
diff --git a/README.md b/README.md
index 3526890..3e5a2e7 100644
--- a/README.md
+++ b/README.md
@@ -38,21 +38,53 @@ DEBUG=routewatch ./routewatch
### Web Interface
-- `GET /` - Redirects to /status
-- `GET /status` - HTML status dashboard
-- `GET /status.json` - JSON statistics
-- `GET /as/{asn}` - AS detail page (HTML)
-- `GET /prefix/{prefix}` - Prefix detail page (HTML)
-- `GET /prefixlength/{length}` - IPv4 prefixes by mask length
-- `GET /prefixlength6/{length}` - IPv6 prefixes by mask length
-- `GET /ip/{ip}` - Redirects to prefix containing the IP
+- `GET /` - Home page (HTML): headline figures, and lookups by AS number and by
+ IP address
+- `GET /status` - HTML status dashboard, which reads `/api/v1/stats`
+- `GET /status.json` - JSON statistics: feed, memory, database and route figures
+- `GET /.well-known/healthcheck.json` - Health check (JSON): 200 when the
+ database answers and the RIS Live feed is connected, otherwise 503;
+ `data.status` is `ok` or `error` and `data.checks` says which check failed
+- `GET /as/{asn}` - AS detail page (HTML): the prefixes the AS originates and
+ its peers
+- `GET /prefix/{prefix}/{length}` - Prefix detail page (HTML), such as
+ `/prefix/192.0.2.0/24`: every live route for the prefix, one per peer, and its
+ origin ASes
+- `GET /prefixlength/{length}` - Up to 500 IPv4 prefixes with that mask length,
+ picked at random, each with its newest route (HTML)
+- `GET /prefixlength6/{length}` - The same for IPv6
+- `GET /ip/{addr}` - The same JSON answer as `/api/v1/ip/{ip}`
+- `GET /ip/` - The same, for the caller's own address, or the one a proxy names
+ in `True-Client-IP`, `X-Real-IP` or `X-Forwarded-For`
### API v1
-- `GET /api/v1/stats` - Detailed statistics with handler metrics
-- `GET /api/v1/ip/{ip}` - Look up AS information for an IP address
-- `GET /api/v1/as/{asn}` - Get prefixes announced by an AS
-- `GET /api/v1/prefix/{prefix}` - Get routes for a specific prefix
+- `GET /api/v1/stats` - The `/status.json` figures plus connection, garbage
+ collector, stream and handler queue statistics
+- `GET /api/v1/ip/{ip}` - Looks up each address: its most specific live route
+ (`netblock`), that route's origin AS with its WHOIS details, and the address's
+ PTR names. `{ip}` is an IP address, a host name (each address it resolves to
+ is looked up), or a comma-separated list of these. `results` has one entry per
+ address with a live route and `errors` one per address or name that failed.
+ The answer is 400 when nothing in `{ip}` is an IP address or a host name that
+ resolves, and 404 when none of the addresses has a live route.
+- `GET /api/v1/as/{asn}` - The AS and the prefixes it originates, IPv4 and IPv6
+ apart, each with when its newest route was updated
+- `GET /api/v1/prefix/{prefix}/{length}` - Every live route for the prefix, one
+ per peer, and the same routes grouped by origin AS in `origins`
+
+A JSON answer is `{"status": "ok", "data": ...}`, or for an error
+`{"status": "error", "error": {"msg": ..., "code": ...}}`, and carries an
+`@meta` object with the time zone, the API version and the time the request
+took. The health check is the exception: its 503 answer is also
+`{"status": "ok", "data": ...}`, with `error` in `data.status`.
+
+A request that runs past its time limit answers 504: the limit is 30 seconds,
+and 4 seconds for the database reads behind `/status.json` and `/api/v1/stats`.
+An AS number, prefix or mask length that does not parse answers 400, as does a
+mask length out of range for its IP version. An AS the database does not hold,
+or a prefix with no live route, answers 404. The HTML pages give these errors as
+plain text.
An origin AS of 0 (`origin_asn` in a route, `asn` in an IP lookup, the key `0`
in a prefix's `origins`) means the route has no known origin: its AS path ends
@@ -76,7 +108,9 @@ routewatch/
│ ├── logger/ # Structured logging wrapper
│ ├── metrics/ # Performance metrics tracking
│ ├── config/ # Configuration management
-│ └── templates/ # HTML templates
+│ ├── templates/ # HTML templates
+│ ├── version/ # Build version information
+│ └── whois/ # WHOIS lookups of AS details
└── pkg/
└── asinfo/ # AS information lookup (public API)
```
@@ -95,7 +129,7 @@ routewatch/
│ │ Streamer │───▶│ Handlers │───▶│ Database │ │
│ │ │ │ │ │ │ │
│ │ RIS Live │ │ - ASHandler │ │ SQLite with │ │
-│ │ WebSocket │ │ - PeerHandler│ │ WAL mode │ │
+│ │ HTTP stream │ │ - PeerHandler│ │ WAL mode │ │
│ │ client │ │ - PrefixHdlr │ │ │ │
│ │ │ │ - PeeringHdlr│ │ Tables: │ │
│ └──────────────┘ └──────────────┘ │ - asns │ │
@@ -131,7 +165,10 @@ routewatch/
- Parses BGP UPDATE messages from JSON stream
- Dispatches to registered handlers based on message type
- Each handler has its own queue with backpressure handling
- - Handlers batch writes for efficiency (25K-30K ops, 1-2s timeout)
+ - Handlers batch writes for efficiency: the AS, peer and prefix handlers
+ write 30,000, 10,000 and 25,000 updates at a time, or after 2, 2 and 1
+ seconds; the peering handler writes the peerings it has collected every 30
+ seconds
4. **Handler Details**
- **ASHandler**: Tracks all ASNs seen in AS paths
@@ -150,7 +187,8 @@ routewatch/
### Key Design Patterns
-- **Batched Writes**: All database operations are batched for performance
+- **Batched Writes**: The handlers write the feed's updates to the database in
+ batches for performance
- **Backpressure**: Probabilistic message dropping when queues exceed 50%
capacity
- **Graceful Shutdown**: 60-second timeout, flushes all pending batches
@@ -169,21 +207,35 @@ routewatch/
```sql
-- Core tables
-asns(id, number, handle, description, first_seen, last_seen)
-prefixes_v4(prefix, first_seen)
-prefixes_v6(prefix, first_seen)
+asns(asn PRIMARY KEY, handle, description,
+ as_name, org_name, org_id, address, country_code, abuse_email,
+ abuse_phone, tech_email, tech_phone, rir, rir_registration_date,
+ rir_last_modified, whois_raw,
+ first_seen, last_seen, whois_updated_at)
+prefixes_v4(prefix PRIMARY KEY, first_seen) WITHOUT ROWID
+prefixes_v6(prefix PRIMARY KEY, first_seen) WITHOUT ROWID
--- Live routing tables (one per IP version)
-live_routes_v4(id, prefix, mask_length, origin_asn, peer_ip, as_path,
- next_hop, last_updated)
-live_routes_v6(id, prefix, mask_length, origin_asn, peer_ip, as_path,
- next_hop, last_updated)
+-- Live routing tables (one per IP version), one route per peer and prefix
+live_routes_v4(id INTEGER PRIMARY KEY, prefix, mask_length, origin_asn,
+ peer_ip, as_path, next_hop, last_updated,
+ UNIQUE(peer_ip, prefix))
+live_routes_v6(id INTEGER PRIMARY KEY, prefix, mask_length, origin_asn,
+ peer_ip, as_path, next_hop, last_updated,
+ UNIQUE(peer_ip, prefix))
-- Relationship tracking
-peerings(id, as_a, as_b, first_seen, last_seen)
-bgp_peers(id, peer_ip, peer_asn, last_message_type, last_seen)
+peerings(id PRIMARY KEY, as_a, as_b, first_seen, last_seen,
+ UNIQUE(as_a, as_b))
+bgp_peers(id PRIMARY KEY, peer_ip UNIQUE, peer_asn, first_seen, last_seen,
+ last_message_type)
+
+-- Nothing in the daemon writes this table
+announcements(id PRIMARY KEY, prefix_id, peer_asn, origin_asn, path,
+ next_hop, timestamp, is_withdrawal)
```
+The indexes are in `internal/database/schema.sql`.
+
## Configuration
Configuration is handled via environment variables and OS-specific paths:
@@ -201,7 +253,7 @@ non-zero exit. An empty variable counts as unset.
State directory (database location):
-- macOS: `~/Library/Application Support/routewatch/`
+- macOS: `~/Library/Application Support/berlin.sneak.app.routewatch/`
- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root, otherwise
`$XDG_DATA_HOME/berlin.sneak.app.routewatch/` (with `XDG_DATA_HOME` unset,
`~/.local/share/berlin.sneak.app.routewatch/`). In the Docker image this is
@@ -301,7 +353,7 @@ make fmt
make lint
# Build
-make
+make build
```
## Entrypoints
@@ -330,8 +382,8 @@ provide:
- `script/docker` — build the Docker image tagged via `script/projectname`
- `script/cibuild` — CI entrypoint: runs `script/bootstrap` and `script/check`,
then builds the image
-- `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must not
- change files), then `script/check`
+- `script/precommit` — pre-commit gate: `go mod tidy` (must not change `go.mod`
+ or `go.sum`) and `go fmt` (writes), then `script/check`
- `script/install-precommit` — install the git pre-commit hook that runs
`script/precommit`
diff --git a/TODO.md b/TODO.md
index ade4f7b..a4f5e9a 100644
--- a/TODO.md
+++ b/TODO.md
@@ -10,28 +10,91 @@
# Status
-pre-1.0. No git tags. The linter and the tests are phases of the Docker build,
-and the Gitea workflow runs `script/cibuild` (the format check of the Go code
-and the markdown, both phases, then the image) on every push. The image sets
-memory ceilings for a 5 GiB container (README "Memory") and the README says how
-to run it under upaas (README "Running under upaas"). A 35-hour run of `3898daa`
-on the live feed peaked at about 1 GiB, without a container memory limit.
+pre-1.0. No git tags. `main` last took `next` on 2026-10-06, through
+https://git.eeqj.de/sneak/routewatch/pulls/6. The linter and the tests are
+phases of the Docker build, and the Gitea workflow runs `script/cibuild` (the
+format check of the Go code and the markdown, both phases, then the image) on
+every push. The image sets memory ceilings for a 5 GiB container (README
+"Memory") and the README says how to run it under upaas (README "Running under
+upaas"). A 25-hour run of `a9b6860` on the live feed peaked at about 0.6 GiB
+(https://git.eeqj.de/sneak/routewatch/issues/3). A run of `825afa2` on the live
+feed found the prefix handler dropping about a quarter of its updates because
+its route writes were slower than the feed
+(https://git.eeqj.de/sneak/routewatch/issues/96); the cheaper writes since then
+are measured by a benchmark, not yet on the live feed.
# Next Step
-`next` waits for sneak to merge it to `main` through
-https://git.eeqj.de/sneak/routewatch/pulls/6. After that, setting routewatch up
-under upaas on fsn1app1 and deploying it are his
-(https://git.eeqj.de/sneak/routewatch/issues/31), and so is the run under a real
-5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3).
+Run `next` on the live feed, as README "Running under upaas" describes, for the
+final check https://git.eeqj.de/sneak/routewatch/issues/96 names: whether the
+prefix handler still drops updates after the first hour. If it does, a new issue
+follows with the numbers. Then an audit of the code at the commit that run
+tested.
# Completed Steps
+- 2026-10-19: README "HTTP Endpoints" lists every route with what it answers,
+ "Database Schema" matches `schema.sql`, and the rest of the README matches the
+ code; this file brought up to date (closes #72)
+- 2026-10-19: route writes are cheaper: the live route and prefix tables lose
+ their random UUID keys and the prefix tables their duplicate index, a route
+ update sets only the columns that change, and every write goes through one
+ connection with a page cache of 1 GiB (README "Memory"). A benchmark writes
+ the prefix handler's largest batch faster than the feed sends it (closes #96)
+- 2026-10-19: the WAL checkpoint, the incremental vacuum and ANALYZE take the
+ database lock every batch write holds, so a vacuum no longer fails with
+ "database is locked" and an ANALYZE no longer makes a batch write lose its
+ updates; ANALYZE runs as `PRAGMA optimize` with an analysis limit (closes #94)
+- 2026-10-19: shutdown stops the streamer first, and each handler writes
+ everything left on its queue before it stops, so the updates waiting there are
+ no longer lost (closes #69)
+- 2026-10-19: a path that ends in an AS set of several members has origin AS 0,
+ no known origin, instead of the set's last member, and the set adds no
+ peerings; the pages show such a route as having no known origin (closes #75)
+- 2026-10-19: `bgp_peer_count` drops to 0 when the last peer goes down (closes
+ #92)
+- 2026-10-19: `bgp_peer_count` counts every peer the feed has heard from, not
+ only those whose session opened since startup, and drops a peer whose session
+ goes down (closes #74)
- 2026-10-19: the shared files are re-vendored from `sneak/prompts` at
`0653579`: golangci-lint v2.14.0 and its configuration, lint and test as
`Dockerfile` phases that keep Go's and golangci-lint's caches, markdown
formatted with prettier, agent guidance in `AGENTS.md`; the code is fixed for
what the new linter configuration finds (closes #52)
+- 2026-10-19: the streamer reads feed lines up to 1 MiB, so a line over 64 KiB
+ no longer ends the connection (closes #76)
+- 2026-10-19: a peer's live routes are removed when its BGP session goes down;
+ `STATE` messages are never dropped for backpressure, and one that finds its
+ queue full waits for room ahead of the handler's other messages (closes #56)
+- 2026-10-19: the prefix-length pages pick their sample with a bounded number of
+ index lookups instead of reading every route of that length, and list each
+ prefix once (closes #66)
+- 2026-10-19: a request past its time limit answers 504 instead of 408, and
+ `/api/v1/stats` answers 504 instead of 500 when its read runs out of time
+ (closes #70)
+- 2026-10-19: the AS, peer and prefix handlers write each batch outside the lock
+ their message handling needs, so messages keep coming off their queues during
+ a write; meanwhile the next batch grows to at most ten batch sizes (closes
+ #65)
+- 2026-10-19: a live route is identified by prefix and peer, so an announcement
+ with a new origin AS replaces the peer's route and a withdrawal removes it
+ whatever its origin AS (closes #55)
+- 2026-10-14: the streamer reconnects when RIS Live ends the stream or sends
+ nothing for 60 seconds, and the health check reports the feed disconnected
+ meanwhile (closes #53)
+- 2026-10-14: the prefix page takes the names of the origin ASes from the prefix
+ query instead of reading every route of each origin AS (closes #67)
+- 2026-10-14: the daemon exits with status 1 when its web server cannot listen
+ on its port, instead of running on without it (closes #73)
+- 2026-10-14: the prefix pages parse the prefix before looking it up: one that
+ does not parse answers 400, and any valid way of writing a prefix finds its
+ routes (closes #71)
+- 2026-10-14: the IP lookup ignores routes shorter than /8 for IPv4 and /16 for
+ IPv6, so a default route is no longer named as an address's owner (closes #57)
+- 2026-10-14: a prefix batch keeps each peer's last update for a prefix, instead
+ of one update per prefix across all peers (closes #54)
+- 2026-10-14: `reconnect_count` in `/api/v1/stats` counts every reconnect
+ (closes #68)
- 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory, seeded
at startup and adjusted on every live-route write, so a request no longer
reads every live route (closes #30)
@@ -116,6 +179,4 @@ under upaas on fsn1app1 and deploying it are his
# Future Steps
-- Production memory under 5 GiB: whether to test under a real 5 GiB container
- limit on fsn1app1 is open for sneak
- (https://git.eeqj.de/sneak/routewatch/issues/3)
+None open.
--
2.54.0
From d2a6e30360e3820f40a16a3198bbefcf6f8c91c9 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Mon, 19 Oct 2026 23:02:00 +0200
Subject: [PATCH 22/45] Percent-decode URL parameters before parsing them
(closes #103)
chi matches routes against the escaped path when a request has one and
hands the handlers their URL parameters still percent-encoded. The home
page's lookup form encodes the address it asks for, so every IPv6 lookup
answered 400, as did a comma-separated address list or a prefix that a
client percent-encoded. The router now matches against the decoded path,
so every handler reads its parameters decoded.
Model: opus-5-5
---
internal/server/handlers_internal_test.go | 86 ++++++++++++++++++++---
internal/server/middleware.go | 15 ++++
internal/server/routes.go | 1 +
3 files changed, 93 insertions(+), 9 deletions(-)
diff --git a/internal/server/handlers_internal_test.go b/internal/server/handlers_internal_test.go
index 8ea16a3..21d9d19 100644
--- a/internal/server/handlers_internal_test.go
+++ b/internal/server/handlers_internal_test.go
@@ -183,14 +183,10 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
}
}
-// TestPrefixDetailHandlersParsePrefix checks that both prefix pages find a
-// stored route whatever valid text form the URL writes its prefix in, and
-// answer 400 for a prefix that does not parse. The tables store the form
-// net/netip prints. Before the fix the URL text was looked up as typed, so
-// "2001:DB8:0:0::/32" was not found although "2001:db8::/32" was stored, and
-// "foo/24" answered 500 (https://git.eeqj.de/sneak/routewatch/issues/71).
-func TestPrefixDetailHandlersParsePrefix(t *testing.T) {
- t.Parallel()
+// newServerWithTestRoutes returns a server on a new database that holds a
+// route for testPrefixV4 and one for 2001:db8::/32.
+func newServerWithTestRoutes(t *testing.T) *Server {
+ t.Helper()
db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
@@ -217,7 +213,22 @@ func TestPrefixDetailHandlersParsePrefix(t *testing.T) {
t.Fatalf("UpsertLiveRouteBatch: %v", err)
}
- s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
+ return New(db, streamer.New(logger.New(), metrics.New()), logger.New(),
+ &config.Config{})
+}
+
+// TestPrefixDetailHandlersParsePrefix checks that both prefix pages find a
+// stored route whatever valid text form the URL writes its prefix in, and
+// answer 400 for a prefix that does not parse. The tables store the form
+// net/netip prints. Before the fix the URL text was looked up as typed, so
+// "2001:DB8:0:0::/32" was not found although "2001:db8::/32" was stored, and
+// "foo/24" answered 500 (https://git.eeqj.de/sneak/routewatch/issues/71). A
+// prefix with its colons percent-encoded, as JavaScript's encodeURIComponent
+// writes it, answered 400 (https://git.eeqj.de/sneak/routewatch/issues/103).
+func TestPrefixDetailHandlersParsePrefix(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTestRoutes(t)
tests := []struct {
path string
@@ -229,6 +240,8 @@ func TestPrefixDetailHandlersParsePrefix(t *testing.T) {
{"/prefix/2001:DB8:0:0::/32", http.StatusOK},
{"/api/v1/prefix/192.0.2.1/24", http.StatusOK},
{"/prefix/192.0.2.1/24", http.StatusOK},
+ {"/api/v1/prefix/2001%3Adb8%3A%3A/32", http.StatusOK},
+ {"/prefix/2001%3Adb8%3A%3A/32", http.StatusOK},
}
for _, tt := range tests {
@@ -243,6 +256,61 @@ func TestPrefixDetailHandlersParsePrefix(t *testing.T) {
}
}
+// TestIPLookupsReadPercentEncodedAddresses checks that the IP lookups find the
+// addresses a URL asks for when it percent-encodes them, as the home page's
+// lookup form does with encodeURIComponent. Before the fix the handler read
+// "2001%3Adb8%3A%3A1" undecoded as a host name, which did not resolve, and
+// answered 400, so the home page could look up no IPv6 address
+// (https://git.eeqj.de/sneak/routewatch/issues/103).
+func TestIPLookupsReadPercentEncodedAddresses(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTestRoutes(t)
+
+ tests := []struct {
+ path string
+ wantIPs string // the addresses of the results, comma-separated
+ }{
+ {"/ip/2001%3Adb8%3A%3A1", "2001:db8::1"},
+ {"/api/v1/ip/192.0.2.1%2C192.0.2.2", "192.0.2.1,192.0.2.2"},
+ }
+
+ for _, tt := range tests {
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, tt.path, nil))
+
+ if rec.Code != http.StatusOK {
+ t.Errorf("%s: status %d, want %d; body %s",
+ tt.path, rec.Code, http.StatusOK, rec.Body)
+
+ continue
+ }
+
+ var body struct {
+ Data IPLookupResponse `json:"data"`
+ }
+
+ err := json.Unmarshal(rec.Body.Bytes(), &body)
+ if err != nil {
+ t.Errorf("%s: decoding the answer %q: %v", tt.path, rec.Body, err)
+
+ continue
+ }
+
+ gotIPs := make([]string, 0, len(body.Data.Results))
+
+ for _, result := range body.Data.Results {
+ gotIPs = append(gotIPs, result.IP)
+ }
+
+ got := strings.Join(gotIPs, ",")
+ if got != tt.wantIPs {
+ t.Errorf("%s: results for %q, want %q", tt.path, got, tt.wantIPs)
+ }
+ }
+}
+
// errASDetailsLookupFailed is what failingASDetailsDB answers every AS details
// lookup with.
var errASDetailsLookupFailed = errors.New("AS details lookup failed")
diff --git a/internal/server/middleware.go b/internal/server/middleware.go
index d2a185d..f088733 100644
--- a/internal/server/middleware.go
+++ b/internal/server/middleware.go
@@ -10,6 +10,8 @@ import (
"net/http"
"sync"
"time"
+
+ "github.com/go-chi/chi/v5"
)
// responseWriter wraps http.ResponseWriter to capture the response
@@ -281,3 +283,16 @@ func RequestLoggerMiddleware(logger *slog.Logger) func(http.Handler) http.Handle
})
}
}
+
+// DecodedPathMiddleware makes the router match routes against the decoded
+// path, so that the handlers read their URL parameters decoded. Without it,
+// chi matches against the escaped path whenever the request has one and hands
+// the handlers their parameters still escaped: /ip/2001%3Adb8%3A%3A1 would
+// look up "2001%3Adb8%3A%3A1".
+func DecodedPathMiddleware(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ chi.RouteContext(r.Context()).RoutePath = r.URL.Path
+
+ next.ServeHTTP(w, r)
+ })
+}
diff --git a/internal/server/routes.go b/internal/server/routes.go
index aca1005..f1fcbc8 100644
--- a/internal/server/routes.go
+++ b/internal/server/routes.go
@@ -12,6 +12,7 @@ func (s *Server) setupRoutes() {
r := chi.NewRouter()
// Middleware
+ r.Use(DecodedPathMiddleware)
r.Use(middleware.RequestID)
r.Use(middleware.RealIP)
r.Use(RequestLoggerMiddleware(s.logger.Logger)) // Structured request logging
--
2.54.0
From 176f9708b82e86c85194c789620b5fc39bd43991 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Mon, 19 Oct 2026 23:19:21 +0200
Subject: [PATCH 23/45] Build and test with a supported Go release (closes
#107)
The test and build stages move from golang:1.24-bookworm, a release
line that no longer gets security fixes, to golang:1.27.2-bookworm,
pinned by digest. The go directive in go.mod moves from 1.24.4 to
1.27.1, the Go in the pinned lint image, so no stage downloads another
toolchain. One test starts its goroutine with sync.WaitGroup.Go, which
the linter asks for once the module targets Go 1.25 or later.
Model: opus-5-5
---
Dockerfile | 8 ++++----
go.mod | 2 +-
internal/database/database_internal_test.go | 7 ++-----
3 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index a8cfb8c..f2f9d83 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,8 +24,8 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
# image ships and the alpine one does not. -count=1 because the mounted
# cache also holds test results, which go test would otherwise replay.
# -short skips the live RIS Live feed test, so the suite needs no network.
-# golang:1.24-bookworm, 2026-09-21
-FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS test
+# golang:1.27.2-bookworm, 2026-10-19
+FROM golang@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 AS test
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
@@ -39,8 +39,8 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
# Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so nothing after them can
# run unless lint and test passed.
-# golang:1.24-bookworm, 2026-09-21
-FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS builder
+# golang:1.27.2-bookworm, 2026-10-19
+FROM golang@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 AS builder
# Install build dependencies (zstd for archive, gcc for CGO/sqlite3)
RUN apt-get update && apt-get install -y --no-install-recommends \
diff --git a/go.mod b/go.mod
index 4d318d3..10d57f5 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module git.eeqj.de/sneak/routewatch
-go 1.24.4
+go 1.27.1
require (
github.com/dustin/go-humanize v1.0.1
diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go
index 92768c9..7a48559 100644
--- a/internal/database/database_internal_test.go
+++ b/internal/database/database_internal_test.go
@@ -564,11 +564,8 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
var wg sync.WaitGroup
- wg.Add(1)
-
- go func() {
- defer wg.Done()
+ wg.Go(func() {
for {
select {
case <-ctx.Done():
@@ -577,7 +574,7 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) {
_ = db.Checkpoint(ctx) // errors are the checkpoint's own to absorb
}
}
- }()
+ })
ts := time.Now().UTC()
for i := range contentionIterations {
--
2.54.0
From a173f4d159dd9559298fcf413a1853ba9da95b97 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Mon, 19 Oct 2026 23:39:28 +0200
Subject: [PATCH 24/45] Keep a peer-down until its route delete succeeds
(closes #101)
When the delete of a peer's live routes after its session went down
failed, the down was dropped and the routes stayed. The prefix handler
now keeps such a peer, once, with the updates it sent after the down,
for the next flush, which retries the delete before it writes them; each
failure is logged. At most the last 25,000 of those updates are kept,
outside the batch, so they do not start a flush. The delete runs in
chunks of at most 10,000 routes per table, taken in prefix order
through the peer's key, each chunk in its own transaction, so the
database lock is released between chunks and a failure undoes at most
one chunk.
Model: opus-5-5
---
README.md | 10 +-
internal/database/counts_internal_test.go | 119 ++++++++++
internal/database/database.go | 168 +++++++++-----
internal/routewatch/prefixhandler.go | 134 ++++++++++--
.../routewatch/prefixhandler_internal_test.go | 206 ++++++++++++++++++
5 files changed, 552 insertions(+), 85 deletions(-)
diff --git a/README.md b/README.md
index 3e5a2e7..05ca2a5 100644
--- a/README.md
+++ b/README.md
@@ -318,8 +318,14 @@ What happens at each limit:
(`peerBatchMaxSize`, about 10 MiB) and 250,000 for the prefix handler
(`prefixBatchMaxSize`, about 60 MiB). At that size the handler takes nothing
more off its queue until the write ends, so the queue fills and the streamer
- drops messages as above. With the batches being written, the three handlers'
- batches take at most about 0.25 GiB of the Go heap's 1.5 GiB.
+ drops messages as above. When the delete of a peer's live routes after its
+ session went down fails, the prefix handler keeps the updates the peer sent
+ after the down for the next write, which retries the delete first; on top of
+ its 250,000 it keeps at most the last 25,000 of them (`prefixBatchSize`, about
+ 6 MiB) and drops and logs the older ones, so while the delete keeps failing
+ the other peers' updates still come off the queue. With the batches being
+ written, the three handlers' batches take at most about 0.25 GiB of the Go
+ heap's 1.5 GiB.
With `DEBUG=routewatch` the daemon logs a `System stats` line every 60 seconds
with the goroutine count and Go memory figures.
diff --git a/internal/database/counts_internal_test.go b/internal/database/counts_internal_test.go
index bbe5577..76f52be 100644
--- a/internal/database/counts_internal_test.go
+++ b/internal/database/counts_internal_test.go
@@ -2,6 +2,7 @@ package database
import (
"context"
+ "fmt"
"slices"
"sync"
"testing"
@@ -494,6 +495,124 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) {
checkOneLiveRoutePerIPVersion(t, db)
}
+// TestDeleteLiveRoutesOfPeerDeletesInChunks gives a peer one IPv4 route more
+// than a chunk holds, two of them for prefixes another peer also has. One chunk
+// deletes peerRouteChunkSize of the peer's routes and commits, leaving the last
+// one. The routes are written again, and DeleteLiveRoutesOfPeer deletes all of
+// them, a chunk and then the last one. After each step the route count and
+// prefix distribution held in memory and those read from the route table are
+// the ones wanted.
+func TestDeleteLiveRoutesOfPeerDeletesInChunks(t *testing.T) {
+ t.Parallel()
+
+ const (
+ peerIP = "192.0.2.1"
+ otherPeerIP = "192.0.2.2"
+ // How many of the peer's prefixes the other peer also has, the first
+ // ones in prefix order.
+ sharedPrefixes = 2
+ )
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+ peerRoutes := make([]*LiveRoute, 0, peerRouteChunkSize+1)
+ otherPeerRoutes := make([]*LiveRoute, 0, sharedPrefixes)
+
+ for i := range peerRouteChunkSize + 1 {
+ prefix := fmt.Sprintf("10.%d.%d.0/24", i/256, i%256)
+
+ route := mkV4Route(t, prefix, 64500, ts)
+ route.PeerIP = peerIP
+ peerRoutes = append(peerRoutes, route)
+
+ if i < sharedPrefixes {
+ route := mkV4Route(t, prefix, 64500, ts)
+ route.PeerIP = otherPeerIP
+ otherPeerRoutes = append(otherPeerRoutes, route)
+ }
+ }
+
+ err = db.UpsertLiveRouteBatch(slices.Concat(peerRoutes, otherPeerRoutes))
+ if err != nil {
+ t.Fatalf("UpsertLiveRouteBatch: %v", err)
+ }
+
+ deletedV4, deletedV6, err := db.deletePeerRouteChunk(peerIP)
+ if err != nil {
+ t.Fatalf("deletePeerRouteChunk: %v", err)
+ }
+
+ if deletedV4 != peerRouteChunkSize || deletedV6 != 0 {
+ t.Errorf("deletePeerRouteChunk deleted (%d, %d) routes, want (%d, 0)",
+ deletedV4, deletedV6, peerRouteChunkSize)
+ }
+
+ // The shared prefixes went with the chunk, so each route left has a
+ // prefix of its own.
+ checkV4RoutesOfOwnPrefixes(t, db, "after one chunk", 1+sharedPrefixes)
+
+ err = db.UpsertLiveRouteBatch(peerRoutes)
+ if err != nil {
+ t.Fatalf("UpsertLiveRouteBatch: %v", err)
+ }
+
+ deleted, err := db.DeleteLiveRoutesOfPeer(peerIP)
+ if err != nil {
+ t.Fatalf("DeleteLiveRoutesOfPeer: %v", err)
+ }
+
+ if deleted != len(peerRoutes) {
+ t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d",
+ deleted, len(peerRoutes))
+ }
+
+ checkV4RoutesOfOwnPrefixes(t, db, "after DeleteLiveRoutesOfPeer",
+ sharedPrefixes)
+}
+
+// checkV4RoutesOfOwnPrefixes checks that the database has wantRoutes IPv4 live
+// routes, each for a /24 no other live route has, both as counted in memory and
+// as read from the route table. when says which step of the test checks.
+func checkV4RoutesOfOwnPrefixes(
+ t *testing.T, db *Database, when string, wantRoutes int,
+) {
+ t.Helper()
+
+ ctx := context.Background()
+
+ stats, err := db.GetStatsContext(ctx)
+ if err != nil {
+ t.Fatalf("GetStatsContext: %v", err)
+ }
+
+ tableRoutes, _, err := db.GetLiveRouteCountsContext(ctx)
+ if err != nil {
+ t.Fatalf("GetLiveRouteCountsContext: %v", err)
+ }
+
+ if stats.IPv4Routes != wantRoutes || tableRoutes != wantRoutes {
+ t.Errorf("%s: IPv4 live routes: %d in memory, %d in the table, want %d",
+ when, stats.IPv4Routes, tableRoutes, wantRoutes)
+ }
+
+ queryDistribution, _, err := db.GetPrefixDistributionContext(ctx)
+ if err != nil {
+ t.Fatalf("GetPrefixDistributionContext: %v", err)
+ }
+
+ want := []PrefixDistribution{{MaskLength: 24, Count: wantRoutes}}
+ assertDistribution(t, when+": IPv4 distribution",
+ stats.IPv4PrefixDistribution, want)
+ assertDistribution(t, when+": IPv4 distribution query",
+ queryDistribution, want)
+}
+
// checkOneLiveRoutePerIPVersion checks that the database has one IPv4 live
// route, for a /24, and one IPv6 live route, for a /32, both as counted in
// memory and as read from the route tables.
diff --git a/internal/database/database.go b/internal/database/database.go
index c4832bb..3d4e89e 100644
--- a/internal/database/database.go
+++ b/internal/database/database.go
@@ -658,48 +658,71 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error {
return nil
}
-// Deleting all live routes of a peer. The first query of each table selects the
-// mask length of each of the peer's routes whose prefix no other peer has a
-// live route for: the delete takes those prefixes out of the prefix
-// distribution.
-const (
- peerOnlyRouteMaskLengthsV4SQL = `SELECT r.mask_length
- FROM live_routes_v4 AS r WHERE r.peer_ip = ?
- AND NOT EXISTS (SELECT 1 FROM live_routes_v4 AS o
- WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)`
- deletePeerLiveRoutesV4SQL = `DELETE FROM live_routes_v4 WHERE peer_ip = ?`
+// peerRouteChunkSize is the most routes of a peer that one transaction of
+// DeleteLiveRoutesOfPeer deletes from each route table. A peer can have a
+// million routes; deleted in one transaction, they would hold the database lock
+// for most of a minute.
+const peerRouteChunkSize = 10000
- peerOnlyRouteMaskLengthsV6SQL = `SELECT r.mask_length
+// Deleting a chunk of a peer's live routes. The first query of each table
+// selects the peer's first routes in prefix order, through the
+// UNIQUE(peer_ip, prefix) key, each with its mask length and whether another
+// peer has a live route for its prefix: the delete takes the prefixes no other
+// peer has out of the prefix distribution. The second query deletes the peer's
+// routes up to the last prefix the first one selected.
+const (
+ peerRouteChunkV4SQL = `SELECT r.prefix, r.mask_length,
+ EXISTS (SELECT 1 FROM live_routes_v4 AS o
+ WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)
+ FROM live_routes_v4 AS r WHERE r.peer_ip = ?
+ ORDER BY r.prefix LIMIT ?`
+ deletePeerRouteChunkV4SQL = `DELETE FROM live_routes_v4
+ WHERE peer_ip = ? AND prefix <= ?`
+
+ peerRouteChunkV6SQL = `SELECT r.prefix, r.mask_length,
+ EXISTS (SELECT 1 FROM live_routes_v6 AS o
+ WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)
FROM live_routes_v6 AS r WHERE r.peer_ip = ?
- AND NOT EXISTS (SELECT 1 FROM live_routes_v6 AS o
- WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)`
- deletePeerLiveRoutesV6SQL = `DELETE FROM live_routes_v6 WHERE peer_ip = ?`
+ ORDER BY r.prefix LIMIT ?`
+ deletePeerRouteChunkV6SQL = `DELETE FROM live_routes_v6
+ WHERE peer_ip = ? AND prefix <= ?`
)
-// deletePeerRoutes deletes all live routes of peerIP from one table, given that
-// table's two statements above. It returns how many routes it deleted and the
-// mask length of each prefix it left with no live route.
+// deletePeerRoutes deletes the first peerRouteChunkSize live routes of peerIP,
+// or as many as it has, from one table, given that table's two statements
+// above. It returns how many routes it deleted and the mask length of each
+// prefix it left with no live route.
func deletePeerRoutes(
- ctx context.Context, tx *loggingTx,
- onlyRouteMaskLengthsSQL, deleteSQL, peerIP string,
+ ctx context.Context, tx *loggingTx, chunkSQL, deleteSQL, peerIP string,
) (int64, []int, error) {
- rows, err := tx.QueryContext(ctx, onlyRouteMaskLengthsSQL, peerIP)
+ rows, err := tx.QueryContext(ctx, chunkSQL, peerIP, peerRouteChunkSize)
if err != nil {
return 0, nil, err
}
defer func() { _ = rows.Close() }()
- var goneMaskLengths []int
+ var (
+ selected int
+ lastPrefix string
+ goneMaskLengths []int
+ )
for rows.Next() {
- var maskLength int
+ var (
+ maskLength int
+ otherPeerHasARoute bool
+ )
- err := rows.Scan(&maskLength)
+ err := rows.Scan(&lastPrefix, &maskLength, &otherPeerHasARoute)
if err != nil {
return 0, nil, err
}
- goneMaskLengths = append(goneMaskLengths, maskLength)
+ selected++
+
+ if !otherPeerHasARoute {
+ goneMaskLengths = append(goneMaskLengths, maskLength)
+ }
}
err = rows.Err()
@@ -707,7 +730,11 @@ func deletePeerRoutes(
return 0, nil, err
}
- res, err := tx.ExecContext(ctx, deleteSQL, peerIP)
+ if selected == 0 {
+ return 0, nil, nil
+ }
+
+ res, err := tx.ExecContext(ctx, deleteSQL, peerIP, lastPrefix)
if err != nil {
return 0, nil, err
}
@@ -721,43 +748,26 @@ func deletePeerRoutes(
}
// DeleteLiveRoutesOfPeer deletes every live route of the peer, IPv4 and IPv6,
-// in one transaction, as when the peer's BGP session goes down. It returns how
-// many routes it deleted.
+// as when the peer's BGP session goes down. It deletes them a chunk at a time
+// with deletePeerRouteChunk, each chunk in a transaction of its own, and
+// releases the database lock between chunks so other writes go on. It returns
+// how many routes it deleted; after an error, the chunks it deleted before stay
+// deleted.
func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
- d.lock("DeleteLiveRoutesOfPeer")
- defer d.unlock()
+ total := 0
- tx, err := d.beginTx()
- if err != nil {
- return 0, fmt.Errorf("failed to begin transaction: %w", err)
+ for {
+ deletedV4, deletedV6, err := d.deletePeerRouteChunk(peerIP)
+ if err != nil {
+ return total, err
+ }
+
+ total += deletedV4 + deletedV6
+
+ if deletedV4 < peerRouteChunkSize && deletedV6 < peerRouteChunkSize {
+ return total, nil
+ }
}
- defer d.rollback(tx)
-
- ctx := context.Background()
-
- deletedV4, goneMaskLengthsV4, err := deletePeerRoutes(ctx, tx,
- peerOnlyRouteMaskLengthsV4SQL, deletePeerLiveRoutesV4SQL, peerIP)
- if err != nil {
- return 0, fmt.Errorf("failed to delete the IPv4 routes of peer %s: %w",
- peerIP, err)
- }
-
- deletedV6, goneMaskLengthsV6, err := deletePeerRoutes(ctx, tx,
- peerOnlyRouteMaskLengthsV6SQL, deletePeerLiveRoutesV6SQL, peerIP)
- if err != nil {
- return 0, fmt.Errorf("failed to delete the IPv6 routes of peer %s: %w",
- peerIP, err)
- }
-
- err = tx.Commit()
- if err != nil {
- return 0, fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- d.counts.addRoutes(-int(deletedV4), -int(deletedV6))
- d.counts.addToDistribution(goneMaskLengthsV4, goneMaskLengthsV6, -1)
-
- return int(deletedV4 + deletedV6), nil
}
// insertPrefixesInTable inserts the prefixes, all of one IP version, that the
@@ -2332,6 +2342,46 @@ func (d *Database) rollback(tx *loggingTx) {
}
}
+// deletePeerRouteChunk deletes, in one transaction, the first
+// peerRouteChunkSize IPv4 and the first peerRouteChunkSize IPv6 live routes of
+// the peer, or as many as it has. It returns how many of each it deleted.
+func (d *Database) deletePeerRouteChunk(peerIP string) (int, int, error) {
+ d.lock("DeleteLiveRoutesOfPeer")
+ defer d.unlock()
+
+ tx, err := d.beginTx()
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to begin transaction: %w", err)
+ }
+ defer d.rollback(tx)
+
+ ctx := context.Background()
+
+ deletedV4, goneMaskLengthsV4, err := deletePeerRoutes(ctx, tx,
+ peerRouteChunkV4SQL, deletePeerRouteChunkV4SQL, peerIP)
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to delete the IPv4 routes of peer %s: %w",
+ peerIP, err)
+ }
+
+ deletedV6, goneMaskLengthsV6, err := deletePeerRoutes(ctx, tx,
+ peerRouteChunkV6SQL, deletePeerRouteChunkV6SQL, peerIP)
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to delete the IPv6 routes of peer %s: %w",
+ peerIP, err)
+ }
+
+ err = tx.Commit()
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to commit transaction: %w", err)
+ }
+
+ d.counts.addRoutes(-int(deletedV4), -int(deletedV6))
+ d.counts.addToDistribution(goneMaskLengthsV4, goneMaskLengthsV6, -1)
+
+ return int(deletedV4), int(deletedV6), nil
+}
+
// routeTimestampRange returns the earliest and latest last_updated across both
// live route tables, or nil values when both tables are empty. Each query reads
// one row from an end of the last_updated index rather than scanning the tables.
diff --git a/internal/routewatch/prefixhandler.go b/internal/routewatch/prefixhandler.go
index c3ae80e..4467e2a 100644
--- a/internal/routewatch/prefixhandler.go
+++ b/internal/routewatch/prefixhandler.go
@@ -28,7 +28,9 @@ const (
// prefixBatchMaxSize is the most prefix updates the batch holds while the
// flush goroutine writes the previous one: ten batches, about 60 MiB
// (README "Memory"). At this size HandleMessage waits until the flush
- // goroutine takes the batch.
+ // goroutine takes the batch. On top of these, the handler holds at most
+ // prefixBatchSize updates, about 6 MiB, that a flush kept for the peers
+ // whose route delete failed (held).
prefixBatchMaxSize = 250000
// prefixBatchTimeout is the maximum time to wait before flushing a batch
@@ -48,13 +50,21 @@ type PrefixHandler struct {
logger *logger.Logger
metrics *metrics.Tracker
- // Batching. mu guards batch and peersDown only: the flush goroutine takes
- // both under mu and writes them after releasing mu, so HandleMessage
- // waits for a database write only while batch is at prefixBatchMaxSize.
+ // Batching. mu guards batch, held and peersDown only: the flush goroutine
+ // takes them under mu and writes them after releasing mu, so
+ // HandleMessage waits for a database write only while batch is at
+ // prefixBatchMaxSize.
mu sync.Mutex
batch []prefixUpdate
- // peersDown holds the peers whose BGP session went down since the batch
- // was taken.
+ // held holds the updates that the peers whose route delete failed in the
+ // last flush sent after their session went down, for the next flush to
+ // write after it retries the delete. They are not in batch, so they do
+ // not count toward a full batch: a delete that keeps failing is retried
+ // at the pace of the flush timer.
+ held []prefixUpdate
+ // peersDown holds, each once, the peers whose BGP session went down
+ // since the batch was taken, and those whose route delete failed in the
+ // last flush.
peersDown []string
// batchTaken is signalled when the flush goroutine takes batch; its L is
// &mu.
@@ -200,17 +210,21 @@ func (h *PrefixHandler) Stop() {
// peerSessionDown notes that the peer's BGP session went down, so that the next
// flush deletes all of the peer's live routes before it writes the batch. The
-// peer's updates still in the batch came before the down, which replaces them,
-// so they are dropped; updates the peer sends later stay in the batch and are
-// written after the delete.
+// peer's updates still in the batch or held came before the down, which
+// replaces them, so they are dropped; updates the peer sends later stay in the
+// batch and are written after the delete. A peer already in peersDown is not
+// added again: one delete removes all of its routes.
func (h *PrefixHandler) peerSessionDown(peer string) {
h.mu.Lock()
defer h.mu.Unlock()
- h.batch = slices.DeleteFunc(h.batch, func(update prefixUpdate) bool {
- return update.peer == peer
- })
- h.peersDown = append(h.peersDown, peer)
+ isPeers := func(update prefixUpdate) bool { return update.peer == peer }
+ h.batch = slices.DeleteFunc(h.batch, isPeers)
+ h.held = slices.DeleteFunc(h.held, isPeers)
+
+ if !slices.Contains(h.peersDown, peer) {
+ h.peersDown = append(h.peersDown, peer)
+ }
}
// flushLoop runs in a goroutine, the only one that writes the batch: it
@@ -238,18 +252,23 @@ func (h *PrefixHandler) flushLoop() {
}
}
-// takeBatch returns the pending updates and the peers whose session went down
-// since the last flush, and leaves an empty batch and no peers in their place
-// for HandleMessage to fill.
+// takeBatch returns the pending updates, the held ones first, and the peers in
+// peersDown, and leaves an empty batch, no held updates and no peers in their
+// place for HandleMessage to fill.
func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) {
h.mu.Lock()
defer h.mu.Unlock()
- if len(h.batch) == 0 && len(h.peersDown) == 0 {
+ if len(h.held) == 0 && len(h.batch) == 0 && len(h.peersDown) == 0 {
return nil, nil
}
batch, peersDown := h.batch, h.peersDown
+ if len(h.held) > 0 {
+ batch = slices.Concat(h.held, h.batch)
+ }
+
+ h.held = nil
h.batch = make([]prefixUpdate, 0, prefixBatchSize)
h.peersDown = nil
h.batchTaken.Broadcast()
@@ -269,8 +288,12 @@ func (h *PrefixHandler) flushBatch() {
batchSize := len(batch)
// The routes of a peer whose session went down are deleted first: the
- // batch holds only the updates the peer sent after the down.
- h.deleteLiveRoutesOfPeers(peersDown)
+ // batch holds only the updates the peer sent after the down. A peer whose
+ // delete failed is kept, with those updates, for the next flush to retry.
+ failedPeers := h.deleteLiveRoutesOfPeers(peersDown)
+ if len(failedPeers) > 0 {
+ batch = h.keepForNextFlush(batch, failedPeers)
+ }
// For each prefix and peer, keep only the last update the feed delivered.
// Updates from different peers for the same prefix are all kept. Arrival
@@ -344,15 +367,21 @@ func (h *PrefixHandler) flushBatch() {
}
// deleteLiveRoutesOfPeers deletes all live routes of each of peers, the peers
-// whose BGP session went down. Most downs come from peers that have no routes,
-// so only a delete that removed routes is logged.
-func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) {
+// whose BGP session went down, and returns those whose delete failed. Most
+// downs come from peers that have no routes, so only a delete that removed
+// routes is logged.
+func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) []string {
+ var failed []string
+
for _, peer := range peers {
deleted, err := h.db.DeleteLiveRoutesOfPeer(peer)
if err != nil {
h.logger.Error(
- "Failed to delete the live routes of a peer whose session went down",
- "peer", peer, "error", err)
+ "Failed to delete the live routes of a peer whose session "+
+ "went down; the next flush retries",
+ "peer", peer, "routes_deleted", deleted, "error", err)
+
+ failed = append(failed, peer)
continue
}
@@ -363,6 +392,63 @@ func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) {
"peer", peer, "routes", deleted)
}
}
+
+ return failed
+}
+
+// keepForNextFlush puts the peers whose route delete failed back in peersDown,
+// and their updates from batch in held, so that the next flush retries each
+// delete before it writes the peer's updates; written now, they would be
+// deleted by the retry. The updates of a peer whose session went down again
+// since the batch was taken came before that down, so they are dropped, as
+// peerSessionDown drops them, and the peer is already in peersDown. It returns
+// the updates of the other peers, for this flush to write.
+//
+// While a delete keeps failing, the held updates would grow with every flush,
+// so only the last prefixBatchSize of them are kept and the older ones are
+// dropped. This never leaves in place a route the peer withdrew: a prefix with
+// a kept update keeps its last one, and the retry deletes all of the peer's
+// routes, so a prefix whose updates were all dropped has no route from the peer
+// until the peer announces it again.
+func (h *PrefixHandler) keepForNextFlush(
+ batch []prefixUpdate, failedPeers []string,
+) []prefixUpdate {
+ var kept, toWrite []prefixUpdate
+
+ for _, update := range batch {
+ if slices.Contains(failedPeers, update.peer) {
+ kept = append(kept, update)
+ } else {
+ toWrite = append(toWrite, update)
+ }
+ }
+
+ h.mu.Lock()
+ defer h.mu.Unlock()
+
+ kept = slices.DeleteFunc(kept, func(update prefixUpdate) bool {
+ return slices.Contains(h.peersDown, update.peer)
+ })
+
+ if len(kept) > prefixBatchSize {
+ h.logger.Error(
+ "Dropped the oldest updates of peers whose route delete failed",
+ "peers", failedPeers, "dropped", len(kept)-prefixBatchSize)
+
+ // A copy, so that the dropped updates are freed.
+ kept = slices.Clone(kept[len(kept)-prefixBatchSize:])
+ }
+
+ // takeBatch left held empty, and only this flush fills it.
+ h.held = kept
+
+ for _, peer := range failedPeers {
+ if !slices.Contains(h.peersDown, peer) {
+ h.peersDown = append(h.peersDown, peer)
+ }
+ }
+
+ return toWrite
}
// writeBatch writes the collected route upserts, route deletions and prefix
diff --git a/internal/routewatch/prefixhandler_internal_test.go b/internal/routewatch/prefixhandler_internal_test.go
index e6a9ad8..0fb851a 100644
--- a/internal/routewatch/prefixhandler_internal_test.go
+++ b/internal/routewatch/prefixhandler_internal_test.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "net/netip"
"slices"
"sync"
"testing"
@@ -560,6 +561,211 @@ func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) {
})
}
+// errTestPeerDelete is the error a test store's failing delete of a peer's
+// routes returns.
+var errTestPeerDelete = errors.New("test: the delete of a peer's routes failed")
+
+// firstPeerDeleteFailsStore is a real database whose first delete of a peer's
+// live routes fails, deleting nothing. deletes counts the deletes asked for.
+type firstPeerDeleteFailsStore struct {
+ *database.Database
+
+ // whileFailing, if not nil, runs during the delete that fails.
+ whileFailing func()
+ deletes int
+}
+
+// DeleteLiveRoutesOfPeer fails the first call and deletes the peer's routes on
+// every later one.
+func (s *firstPeerDeleteFailsStore) DeleteLiveRoutesOfPeer(
+ peerIP string,
+) (int, error) {
+ s.deletes++
+ if s.deletes > 1 {
+ return s.Database.DeleteLiveRoutesOfPeer(peerIP)
+ }
+
+ if s.whileFailing != nil {
+ s.whileFailing()
+ }
+
+ return 0, errTestPeerDelete
+}
+
+// TestPrefixHandlerRetriesAFailedDeleteOfAPeersRoutes runs, through the prefix
+// handler into a database whose first delete of a peer's routes fails, one flush
+// after another: a peer's announcement, then its session going down and a new
+// announcement from the peer, then nothing. The failed delete must be retried
+// on the next flush, and the announcement the peer sent after the down written
+// only after that retry, so it is kept. In the other cases the peer's session
+// goes down again, and the peer announces another prefix, while the delete
+// fails or after the flush whose delete failed: the announcement sent between
+// the two downs must not be written, and the next flush must delete the peer's
+// routes once, not once for the retry and once for the down.
+func TestPrefixHandlerRetriesAFailedDeleteOfAPeersRoutes(t *testing.T) {
+ t.Parallel()
+
+ const (
+ beforeDown = "192.0.2.0/24"
+ afterDown = "198.51.100.0/24"
+ afterSecondDown = "203.0.113.0/24"
+ maskLength = 24
+ // The delete that fails and its retry, which is also the delete for
+ // a second down.
+ wantDeletes = 2
+ )
+
+ tests := []struct {
+ name string
+ // Handled while the delete fails, if not nil.
+ whileFailing []*ristypes.RISMessage
+ // Handled after the flush whose delete fails, if not nil.
+ afterFailing []*ristypes.RISMessage
+ // The peers left with a live route for each prefix.
+ wantPeers map[string][]string
+ }{
+ {
+ name: "the delete fails once",
+ wantPeers: map[string][]string{
+ beforeDown: nil, afterDown: {testPeerIP}, afterSecondDown: nil,
+ },
+ },
+ {
+ name: "the session goes down again while the delete fails",
+ whileFailing: []*ristypes.RISMessage{
+ sessionDown(), peerAnnouncement(testPeerIP, afterSecondDown),
+ },
+ wantPeers: map[string][]string{
+ beforeDown: nil, afterDown: nil, afterSecondDown: {testPeerIP},
+ },
+ },
+ {
+ name: "the session goes down again after the delete failed",
+ afterFailing: []*ristypes.RISMessage{
+ sessionDown(), peerAnnouncement(testPeerIP, afterSecondDown),
+ },
+ wantPeers: map[string][]string{
+ beforeDown: nil, afterDown: nil, afterSecondDown: {testPeerIP},
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ db := newTestDatabase(t)
+ store := &firstPeerDeleteFailsStore{Database: db}
+
+ // Built without NewPrefixHandler's flush timer, so the test
+ // flushes itself.
+ h := &PrefixHandler{db: store, logger: logger.New()}
+ store.whileFailing = func() {
+ for _, msg := range tt.whileFailing {
+ h.HandleMessage(msg)
+ }
+ }
+
+ h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown))
+ h.flushBatch()
+ h.HandleMessage(sessionDown())
+ h.HandleMessage(peerAnnouncement(testPeerIP, afterDown))
+ h.flushBatch() // the delete fails
+
+ for _, msg := range tt.afterFailing {
+ h.HandleMessage(msg)
+ }
+
+ h.flushBatch()
+
+ if store.deletes != wantDeletes {
+ t.Errorf("the store was asked for %d deletes of the peer's "+
+ "routes, want %d", store.deletes, wantDeletes)
+ }
+
+ assertPeers(t, db, tt.wantPeers)
+ assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{
+ {MaskLength: maskLength, Count: 1},
+ })
+ })
+ }
+}
+
+// peerDeleteAlwaysFailsStore is a real database whose every delete of a peer's
+// live routes fails, deleting nothing.
+type peerDeleteAlwaysFailsStore struct {
+ *database.Database
+}
+
+// DeleteLiveRoutesOfPeer fails.
+func (peerDeleteAlwaysFailsStore) DeleteLiveRoutesOfPeer(string) (int, error) {
+ return 0, errTestPeerDelete
+}
+
+// TestPrefixHandlerKeepsAtMostABatchOfUpdatesWhileADeleteKeepsFailing runs,
+// through the prefix handler into a database whose every delete of a peer's
+// routes fails, a peer's session going down and then three flushes. Before each
+// flush the peer announces prefixBatchSize new prefixes and another peer
+// announces one. After each flush the handler must hold only the peer's last
+// prefixBatchSize updates, not every update since the down, and the other
+// peer's route must be written. After the last flush, one more update must not
+// wake the flush goroutine: held updates do not count toward a full batch.
+func TestPrefixHandlerKeepsAtMostABatchOfUpdatesWhileADeleteKeepsFailing(
+ t *testing.T,
+) {
+ t.Parallel()
+
+ const otherPeerIP = "2001:db8:ffff::2"
+
+ // One prefix the other peer announces before each flush.
+ otherPeersPrefixes := []string{testV4Prefix, testV6Prefix, "2001:db8:2::/48"}
+
+ db := newTestDatabase(t)
+
+ // Built without NewPrefixHandler's flush timer, so the test flushes itself.
+ h := &PrefixHandler{
+ db: peerDeleteAlwaysFailsStore{Database: db}, logger: logger.New(),
+ }
+ h.HandleMessage(sessionDown())
+
+ // The peer announces a /32 for each address from 10.0.0.0 on, each
+ // address once.
+ addr := netip.MustParseAddr("10.0.0.0")
+
+ for flush, otherPeersPrefix := range otherPeersPrefixes {
+ prefixes := make([]string, prefixBatchSize)
+ for i := range prefixes {
+ prefixes[i] = netip.PrefixFrom(addr, addr.BitLen()).String()
+ addr = addr.Next()
+ }
+
+ h.HandleMessage(peerAnnouncement(testPeerIP, prefixes...))
+ h.HandleMessage(peerAnnouncement(otherPeerIP, otherPeersPrefix))
+ h.flushBatch()
+
+ kept := make([]string, 0, len(h.held))
+ for _, update := range h.held {
+ kept = append(kept, update.prefix)
+ }
+
+ if !slices.Equal(kept, prefixes) {
+ t.Fatalf("after flush %d the handler holds %d updates, not just "+
+ "the peer's last %d", flush+1, len(kept), len(prefixes))
+ }
+
+ assertPeers(t, db, map[string][]string{otherPeersPrefix: {otherPeerIP}})
+ }
+
+ // A new channel, so that a wake-up in it comes from the update below.
+ h.batchFullCh = make(chan struct{}, 1)
+ h.HandleMessage(peerAnnouncement(otherPeerIP, testV4Prefix))
+
+ if len(h.batchFullCh) > 0 {
+ t.Error("one update after a flush that held a batch of updates " +
+ "woke the flush goroutine")
+ }
+}
+
// newTestDatabase opens a database in a new temporary directory. It is closed
// when the test ends.
func newTestDatabase(t *testing.T) *database.Database {
--
2.54.0
From abdc327c4c60e262aec3dfd0637348d449b18685 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Tue, 20 Oct 2026 00:02:34 +0200
Subject: [PATCH 25/45] Retry a WHOIS lookup that failed instead of storing it
as complete (closes #105)
A lookup whose RIR query failed, or whose RIR answer carried none of the
WHOIS fields (a refusal of service, for one), returned IANA's answer as
a success. The fetcher stored it as complete with blank fields, did not
look the AS up again for 30 days, and shortened its interval. Such a
lookup is now an error: the fetcher backs off and leaves the row as it
was. The fields are read from the RIR's answer alone. An AS IANA refers
to no RIR is stored with no WHOIS data, so it is not looked up again
until it is stale, counting as neither a success nor a failure. Tests
reach fake WHOIS servers through the client's DialContext.
Model: opus-5-5
---
internal/routewatch/asnfetcher.go | 18 +-
.../routewatch/asnfetcher_internal_test.go | 157 +++++++++++++++
internal/whois/whois.go | 104 +++++-----
internal/whois/whois_test.go | 189 ++++++++++++++++++
4 files changed, 416 insertions(+), 52 deletions(-)
create mode 100644 internal/routewatch/asnfetcher_internal_test.go
create mode 100644 internal/whois/whois_test.go
diff --git a/internal/routewatch/asnfetcher.go b/internal/routewatch/asnfetcher.go
index 7320276..5dbf1eb 100644
--- a/internal/routewatch/asnfetcher.go
+++ b/internal/routewatch/asnfetcher.go
@@ -264,7 +264,10 @@ func (f *ASNFetcher) fetchNextStale(ctx context.Context) {
}
// fetchAndUpdate performs a WHOIS lookup and updates the database.
-// Returns true if successful.
+// Returns true if successful. A failed lookup leaves the AS's row as it was,
+// so the AS stays stale. An AS IANA refers to no RIR is stored with no
+// WHOIS data, so it is not looked up again until it is stale; that counts
+// as neither a success nor a failure.
func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
ctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
@@ -272,6 +275,19 @@ func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
f.logger.Info("Fetching WHOIS data", "asn", asn)
info, err := f.whoisClient.LookupASN(ctx, asn)
+ if errors.Is(err, whois.ErrNoReferral) {
+ f.logger.Info("IANA refers the AS to no RIR", "asn", asn)
+
+ err = f.db.UpdateASNWHOIS(ctx, &database.ASNWHOISUpdate{ASN: asn})
+ if err != nil {
+ f.logger.Error("Failed to update ASN WHOIS data",
+ "asn", asn, "error", err)
+ f.recordFailure()
+ }
+
+ return false
+ }
+
if err != nil {
f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err)
f.recordFailure()
diff --git a/internal/routewatch/asnfetcher_internal_test.go b/internal/routewatch/asnfetcher_internal_test.go
new file mode 100644
index 0000000..d929068
--- /dev/null
+++ b/internal/routewatch/asnfetcher_internal_test.go
@@ -0,0 +1,157 @@
+package routewatch
+
+import (
+ "bufio"
+ "context"
+ "errors"
+ "io"
+ "net"
+ "sync"
+ "syscall"
+ "testing"
+ "time"
+
+ "git.eeqj.de/sneak/routewatch/internal/database"
+ "git.eeqj.de/sneak/routewatch/internal/logger"
+)
+
+// The WHOIS servers a lookup asks: IANA first, then the RIR IANA refers it
+// to.
+const (
+ ianaWHOISServer = "whois.iana.org:43"
+ ripeWHOISServer = "whois.ripe.net:43"
+)
+
+// ianaRefersToRIPE is IANA's answer for an AS in a block it gave to RIPE.
+const ianaRefersToRIPE = `refer: whois.ripe.net
+
+as-block: 64496-64511
+organisation: Assigned by RIPE NCC
+changed: 2002-09-01
+source: IANA
+`
+
+// ripeAccessDenied is RIPE's answer to a host it no longer serves.
+const ripeAccessDenied = `% This is the RIPE Database query service.
+%ERROR:201: access denied for 192.0.2.1
+`
+
+// ianaPrivateUseBlock is IANA's answer for a private-use AS: its block and
+// no referral.
+const ianaPrivateUseBlock = `as-block: 64512-65534
+organisation: Reserved for Private Use
+source: IANA
+`
+
+// newFetcherWithWHOISAnswers returns an ASN fetcher on a new database that
+// holds testASNA, never looked up. The fetcher's WHOIS client connects to a
+// fake server for each address in answers, which reads the query and sends
+// answers[address]; connecting to any other address is refused.
+func newFetcherWithWHOISAnswers(
+ t *testing.T, answers map[string]string,
+) (*ASNFetcher, *database.Database) {
+ t.Helper()
+
+ db := newTestDatabase(t)
+
+ _, err := db.GetOrCreateASN(testASNA, time.Now())
+ if err != nil {
+ t.Fatalf("GetOrCreateASN: %v", err)
+ }
+
+ var servers sync.WaitGroup
+
+ t.Cleanup(servers.Wait)
+
+ f := NewASNFetcher(db, logger.New().Logger)
+ f.whoisClient.DialContext = func(
+ _ context.Context, _, address string,
+ ) (net.Conn, error) {
+ answer, ok := answers[address]
+ if !ok {
+ return nil, syscall.ECONNREFUSED
+ }
+
+ client, server := net.Pipe()
+
+ servers.Go(func() {
+ defer func() { _ = server.Close() }()
+
+ _, _ = bufio.NewReader(server).ReadString('\n')
+ _, _ = io.WriteString(server, answer)
+ })
+
+ return client, nil
+ }
+
+ return f, db
+}
+
+// TestFetchAndUpdateKeepsTheRowWhenTheLookupFails looks up an AS IANA refers
+// to RIPE when RIPE's server refuses the connection, and when it answers only
+// with an error. The AS must stay never looked up, and the fetcher must back
+// off.
+func TestFetchAndUpdateKeepsTheRowWhenTheLookupFails(t *testing.T) {
+ t.Parallel()
+
+ for name, answers := range map[string]map[string]string{
+ "connection refused": {ianaWHOISServer: ianaRefersToRIPE},
+ "only an error": {
+ ianaWHOISServer: ianaRefersToRIPE,
+ ripeWHOISServer: ripeAccessDenied,
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+
+ f, db := newFetcherWithWHOISAnswers(t, answers)
+
+ if f.fetchAndUpdate(t.Context(), testASNA) {
+ t.Error("fetchAndUpdate reported success")
+ }
+
+ stats, err := db.GetWHOISStats(t.Context(), whoisStaleThreshold)
+ if err != nil {
+ t.Fatalf("GetWHOISStats: %v", err)
+ }
+
+ if stats.NeverFetched != 1 {
+ t.Errorf("never looked up = %d, want 1: the failed lookup "+
+ "was stored", stats.NeverFetched)
+ }
+
+ got := f.GetStats()
+ if got.CurrentInterval != baseInterval*backoffMultiplier ||
+ got.ConsecutiveFails != 1 {
+ t.Errorf("interval %v after %d failures, want %v after 1",
+ got.CurrentInterval, got.ConsecutiveFails,
+ baseInterval*backoffMultiplier)
+ }
+ })
+ }
+}
+
+// TestFetchAndUpdateMarksAnASWithoutReferral looks up a private-use AS, which
+// IANA refers to no RIR. The AS must not be looked up again until it is
+// stale, and the lookup counts as neither a success nor a failure.
+func TestFetchAndUpdateMarksAnASWithoutReferral(t *testing.T) {
+ t.Parallel()
+
+ f, db := newFetcherWithWHOISAnswers(t,
+ map[string]string{ianaWHOISServer: ianaPrivateUseBlock})
+
+ f.fetchAndUpdate(t.Context(), testASNA)
+
+ asn, err := db.GetNextStaleASN(t.Context(), whoisStaleThreshold)
+ if !errors.Is(err, database.ErrNoStaleASN) {
+ t.Errorf("GetNextStaleASN = %d, %v; want ErrNoStaleASN", asn, err)
+ }
+
+ got := f.GetStats()
+ if got.CurrentInterval != baseInterval || got.SuccessesLastHour != 0 ||
+ got.ErrorsLastHour != 0 {
+ t.Errorf("interval %v, %d successes, %d errors; want %v, 0, 0",
+ got.CurrentInterval, got.SuccessesLastHour, got.ErrorsLastHour,
+ baseInterval)
+ }
+}
diff --git a/internal/whois/whois.go b/internal/whois/whois.go
index 3713c9f..9e94a92 100644
--- a/internal/whois/whois.go
+++ b/internal/whois/whois.go
@@ -4,6 +4,7 @@ package whois
import (
"bufio"
"context"
+ "errors"
"fmt"
"net"
"regexp"
@@ -43,6 +44,17 @@ const (
RIRAFRNIC = "AFRINIC"
)
+// Errors LookupASN returns when a WHOIS answer has nothing to store.
+var (
+ // ErrNoReferral is returned when IANA refers the AS to no RIR, as it
+ // does for a reserved, private-use or unallocated AS number.
+ ErrNoReferral = errors.New("IANA refers the AS to no RIR")
+
+ errNoASBlock = errors.New("IANA answered with neither a referral " +
+ "nor an AS block")
+ errNoFields = errors.New("RIR answered with no WHOIS fields")
+)
+
// ASNInfo contains parsed WHOIS information for an ASN.
type ASNInfo struct {
ASN int
@@ -63,20 +75,24 @@ type ASNInfo struct {
// Client performs WHOIS lookups for ASNs.
type Client struct {
- // Dialer for creating connections (can be overridden for testing)
- dialer *net.Dialer
+ // DialContext opens the connection to a WHOIS server, given as
+ // host:port. NewClient sets a net.Dialer's; tests replace it to reach
+ // local servers instead.
+ DialContext func(ctx context.Context, network, address string) (net.Conn, error)
}
// NewClient creates a new WHOIS client.
func NewClient() *Client {
- return &Client{
- dialer: &net.Dialer{
- Timeout: dialTimeout,
- },
- }
+ dialer := &net.Dialer{Timeout: dialTimeout}
+
+ return &Client{DialContext: dialer.DialContext}
}
-// LookupASN queries WHOIS for the given ASN and returns parsed information.
+// LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and
+// returns what the RIR's answer says. It returns ErrNoReferral when IANA
+// refers the AS to no RIR, and another error when a query fails, when IANA's
+// answer has neither a referral nor the AS's block, or when the RIR's answer
+// carries none of ASNInfo's fields, as a refusal of service does.
func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
// Query IANA first to find the authoritative RIR
query := fmt.Sprintf("AS%d", asn)
@@ -89,32 +105,46 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
// Determine RIR from IANA response
rir, whoisServer := c.parseIANAReferral(ianaResp)
if whoisServer == "" {
- // No referral, try to parse what we have
- return c.parseResponse(asn, rir, ianaResp), nil
+ // IANA names the block of an AS it refers to no RIR. An answer
+ // without that, such as an empty one, says nothing about the AS.
+ if !strings.Contains(ianaResp, "as-block:") {
+ return nil, fmt.Errorf("%s: %w", query, errNoASBlock)
+ }
+
+ return nil, fmt.Errorf("%s: %w", query, ErrNoReferral)
}
// Query the authoritative RIR
rirResp, err := c.query(ctx, whoisServer, query)
if err != nil {
- // Return partial data from IANA if RIR query fails
- info := c.parseResponse(asn, rir, ianaResp)
- info.RawResponse = fmt.Sprintf("%s\n--- RIR query failed: %v ---\n",
- ianaResp, err)
-
- return info, nil
+ return nil, fmt.Errorf("RIR query failed: %w", err)
}
- // Combine responses and parse
- fullResponse := ianaResp + "\n" + rirResp
- info := c.parseResponse(asn, rir, fullResponse)
- info.RawResponse = fullResponse
+ // The fields come from the RIR's answer alone: IANA's describes the
+ // block the AS is in.
+ info := c.parseResponse(asn, rir, rirResp)
+ if info.isEmpty() {
+ return nil, fmt.Errorf("%s from %s: %w", query, whoisServer, errNoFields)
+ }
+
+ info.RawResponse = ianaResp + "\n" + rirResp
return info, nil
}
+// isEmpty reports whether info holds none of the fields a WHOIS answer
+// fills, as when the answer is only comment and error lines.
+func (info *ASNInfo) isEmpty() bool {
+ return info.ASName == "" && info.OrgName == "" && info.OrgID == "" &&
+ info.Address == "" && info.CountryCode == "" &&
+ info.AbuseEmail == "" && info.AbusePhone == "" &&
+ info.TechEmail == "" && info.TechPhone == "" &&
+ info.RegDate == nil && info.LastMod == nil
+}
+
// query performs a raw WHOIS query to the specified server.
func (c *Client) query(ctx context.Context, server, query string) (string, error) {
- conn, err := c.dialer.DialContext(ctx, "tcp", server)
+ conn, err := c.DialContext(ctx, "tcp", server)
if err != nil {
return "", fmt.Errorf("dial %s: %w", server, err)
}
@@ -155,9 +185,8 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
}
// parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
+// The server is "" when the response has no referral.
func (c *Client) parseIANAReferral(response string) (string, string) {
- var rir string
-
for line := range strings.SplitSeq(response, "\n") {
line = strings.TrimSpace(line)
@@ -186,36 +215,9 @@ func (c *Client) parseIANAReferral(response string) (string, string) {
return "", server
}
}
-
- // Also check organisation line for RIR hints
- if strings.HasPrefix(strings.ToLower(line), "organisation:") {
- orgRIR := rirInOrganisationLine(strings.ToLower(line))
- if orgRIR != "" {
- rir = orgRIR
- }
- }
}
- return rir, ""
-}
-
-// rirInOrganisationLine returns the RIR a lowercased IANA "organisation:"
-// line names, or "" when it names none.
-func rirInOrganisationLine(org string) string {
- switch {
- case strings.Contains(org, "arin"):
- return RIRARIN
- case strings.Contains(org, "ripe"):
- return RIRRIPE
- case strings.Contains(org, "apnic"):
- return RIRAPNIC
- case strings.Contains(org, "lacnic"):
- return RIRLACNIC
- case strings.Contains(org, "afrinic"):
- return RIRAFRNIC
- default:
- return ""
- }
+ return "", ""
}
// parseResponse extracts ASN information from a WHOIS response.
diff --git a/internal/whois/whois_test.go b/internal/whois/whois_test.go
new file mode 100644
index 0000000..8d3a150
--- /dev/null
+++ b/internal/whois/whois_test.go
@@ -0,0 +1,189 @@
+package whois_test
+
+import (
+ "bufio"
+ "context"
+ "errors"
+ "io"
+ "net"
+ "sync"
+ "syscall"
+ "testing"
+ "time"
+
+ "git.eeqj.de/sneak/routewatch/internal/whois"
+)
+
+// The servers LookupASN asks: IANA first, then the RIR IANA refers it to.
+const (
+ ianaServer = "whois.iana.org:43"
+ ripeServer = "whois.ripe.net:43"
+)
+
+// testASN is the AS the tests look up.
+const testASN = 3333
+
+// ianaReferral is IANA's answer for an AS in a block it gave to RIPE.
+const ianaReferral = `% IANA WHOIS server
+
+refer: whois.ripe.net
+
+as-block: 3154-3353
+organisation: Assigned by RIPE NCC
+whois: whois.ripe.net
+changed: 2002-09-01
+source: IANA
+`
+
+// ripeRecord is RIPE's answer for the AS.
+const ripeRecord = `% This is the RIPE Database query service.
+
+aut-num: AS3333
+as-name: RIPE-NCC-AS
+org: ORG-RIEN1-RIPE
+created: 2002-09-17T13:08:32Z
+last-modified: 2024-01-09T08:55:41Z
+source: RIPE
+`
+
+// ripeRefusal is RIPE's answer to a host it no longer serves.
+const ripeRefusal = `% This is the RIPE Database query service.
+% The objects are in RPSL format.
+%
+%ERROR:201: access denied for 192.0.2.1
+%
+% Sorry, access from your host has been permanently
+% denied because of a repeated excessive querying.
+`
+
+// ianaPrivateUse is IANA's answer for a private-use AS: its block and no
+// referral.
+const ianaPrivateUse = `% IANA WHOIS server
+
+as-block: 64512-65534
+organisation: Reserved for Private Use
+descr: Reserved for Private Use
+source: IANA
+`
+
+// ianaNoObject is IANA's answer when it has nothing for the query.
+const ianaNoObject = `% IANA WHOIS server
+% This query returned 0 objects.
+`
+
+// fakeServers returns a DialContext for a whois.Client that connects to a
+// fake server for each address in answers, which reads the query and sends
+// answers[address]. Connecting to any other address is refused.
+func fakeServers(
+ t *testing.T, answers map[string]string,
+) func(context.Context, string, string) (net.Conn, error) {
+ t.Helper()
+
+ var servers sync.WaitGroup
+
+ t.Cleanup(servers.Wait)
+
+ return func(_ context.Context, _, address string) (net.Conn, error) {
+ answer, ok := answers[address]
+ if !ok {
+ return nil, syscall.ECONNREFUSED
+ }
+
+ client, server := net.Pipe()
+
+ servers.Go(func() {
+ defer func() { _ = server.Close() }()
+
+ _, _ = bufio.NewReader(server).ReadString('\n')
+ _, _ = io.WriteString(server, answer)
+ })
+
+ return client, nil
+ }
+}
+
+// lookUp looks testASN up with fake servers sending answers.
+func lookUp(
+ t *testing.T, answers map[string]string,
+) (*whois.ASNInfo, error) {
+ t.Helper()
+
+ client := whois.NewClient()
+ client.DialContext = fakeServers(t, answers)
+
+ return client.LookupASN(t.Context(), testASN)
+}
+
+// TestLookupASNReadsTheRIRAnswer looks up an AS IANA refers to RIPE. The
+// fields come from RIPE's answer, the last-modified date included: IANA's
+// "changed:" line dates its block, not the AS.
+func TestLookupASNReadsTheRIRAnswer(t *testing.T) {
+ t.Parallel()
+
+ info, err := lookUp(t, map[string]string{
+ ianaServer: ianaReferral,
+ ripeServer: ripeRecord,
+ })
+ if err != nil {
+ t.Fatalf("LookupASN: %v", err)
+ }
+
+ if info.ASName != "RIPE-NCC-AS" || info.OrgID != "ORG-RIEN1-RIPE" ||
+ info.RIR != whois.RIRRIPE {
+ t.Errorf("LookupASN = AS name %q, org %q, RIR %q; "+
+ "want RIPE-NCC-AS, ORG-RIEN1-RIPE, RIPE",
+ info.ASName, info.OrgID, info.RIR)
+ }
+
+ wantLastMod := time.Date(2024, 1, 9, 8, 55, 41, 0, time.UTC)
+ if info.LastMod == nil || !info.LastMod.Equal(wantLastMod) {
+ t.Errorf("LookupASN last modified = %v, want %v",
+ info.LastMod, wantLastMod)
+ }
+}
+
+// TestLookupASNFailsWhenTheRIRGivesNoAnswer looks up an AS IANA refers to
+// RIPE when RIPE's server refuses the connection, and when it answers only
+// with an error. Neither is a lookup that found the AS.
+func TestLookupASNFailsWhenTheRIRGivesNoAnswer(t *testing.T) {
+ t.Parallel()
+
+ for name, answers := range map[string]map[string]string{
+ "connection refused": {ianaServer: ianaReferral},
+ "only an error": {ianaServer: ianaReferral, ripeServer: ripeRefusal},
+ } {
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+
+ _, err := lookUp(t, answers)
+ if err == nil || errors.Is(err, whois.ErrNoReferral) {
+ t.Errorf("LookupASN error = %v, want one that is not "+
+ "ErrNoReferral", err)
+ }
+ })
+ }
+}
+
+// TestLookupASNReportsNoReferral looks up a private-use AS, which IANA
+// refers to no RIR.
+func TestLookupASNReportsNoReferral(t *testing.T) {
+ t.Parallel()
+
+ _, err := lookUp(t, map[string]string{ianaServer: ianaPrivateUse})
+ if !errors.Is(err, whois.ErrNoReferral) {
+ t.Errorf("LookupASN error = %v, want ErrNoReferral", err)
+ }
+}
+
+// TestLookupASNFailsWhenIANAAnswersNoBlock looks up an AS when IANA answers
+// with neither a referral nor the AS's block. That is not IANA saying no
+// RIR holds the AS.
+func TestLookupASNFailsWhenIANAAnswersNoBlock(t *testing.T) {
+ t.Parallel()
+
+ _, err := lookUp(t, map[string]string{ianaServer: ianaNoObject})
+ if err == nil || errors.Is(err, whois.ErrNoReferral) {
+ t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral",
+ err)
+ }
+}
--
2.54.0
From 69d477159c67634758f66645d1ddce19201323f8 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Tue, 20 Oct 2026 00:15:14 +0200
Subject: [PATCH 26/45] Cap WHOIS answers and follow referrals only to WHOIS
servers (closes #106)
The client kept every line a WHOIS server sent until the server closed
the connection or 30 seconds passed, so a server that kept sending
could fill memory. It now reads at most 1 MiB of an answer, and an
answer that reaches that is an error. A "refer:" line in IANA's answer
naming a server none of the five RIRs run was dialled as given, on any
host and port; it is now an error, so the client only connects to IANA
and the five RIR servers. Both errors are failed lookups, which the
fetcher retries later.
Model: opus-5-5
---
internal/whois/whois.go | 58 +++++++++++++++++++++++-------------
internal/whois/whois_test.go | 57 +++++++++++++++++++++++++++++++++++
2 files changed, 95 insertions(+), 20 deletions(-)
diff --git a/internal/whois/whois.go b/internal/whois/whois.go
index 9e94a92..92df9fa 100644
--- a/internal/whois/whois.go
+++ b/internal/whois/whois.go
@@ -6,6 +6,7 @@ import (
"context"
"errors"
"fmt"
+ "io"
"net"
"regexp"
"strings"
@@ -19,6 +20,10 @@ const (
writeTimeout = 5 * time.Second
)
+// maxAnswerSize caps the bytes read of one WHOIS answer: a real one is a
+// few KiB, so one this large is from a server that does not stop sending.
+const maxAnswerSize = 1 << 20
+
// Parsing constants.
const (
keyValueParts = 2 // Expected parts when splitting "key: value"
@@ -52,7 +57,9 @@ var (
errNoASBlock = errors.New("IANA answered with neither a referral " +
"nor an AS block")
- errNoFields = errors.New("RIR answered with no WHOIS fields")
+ errNoFields = errors.New("RIR answered with no WHOIS fields")
+ errUnknownServer = errors.New("IANA refers the AS to a server no RIR runs")
+ errAnswerTooLarge = errors.New("WHOIS answer reached the size cap")
)
// ASNInfo contains parsed WHOIS information for an ASN.
@@ -91,8 +98,9 @@ func NewClient() *Client {
// LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and
// returns what the RIR's answer says. It returns ErrNoReferral when IANA
// refers the AS to no RIR, and another error when a query fails, when IANA's
-// answer has neither a referral nor the AS's block, or when the RIR's answer
-// carries none of ASNInfo's fields, as a refusal of service does.
+// answer has neither a referral nor the AS's block, when IANA refers the AS
+// to a server none of the five RIRs run, or when the RIR's answer carries
+// none of ASNInfo's fields, as a refusal of service does.
func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
// Query IANA first to find the authoritative RIR
query := fmt.Sprintf("AS%d", asn)
@@ -103,7 +111,11 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
}
// Determine RIR from IANA response
- rir, whoisServer := c.parseIANAReferral(ianaResp)
+ rir, whoisServer, err := c.parseIANAReferral(ianaResp)
+ if err != nil {
+ return nil, fmt.Errorf("%s: %w", query, err)
+ }
+
if whoisServer == "" {
// IANA names the block of an AS it refers to no RIR. An answer
// without that, such as an empty one, says nothing about the AS.
@@ -142,7 +154,8 @@ func (info *ASNInfo) isEmpty() bool {
info.RegDate == nil && info.LastMod == nil
}
-// query performs a raw WHOIS query to the specified server.
+// query performs a raw WHOIS query to the specified server. An answer that
+// reaches maxAnswerSize is an error.
func (c *Client) query(ctx context.Context, server, query string) (string, error) {
conn, err := c.DialContext(ctx, "tcp", server)
if err != nil {
@@ -170,7 +183,11 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
var sb strings.Builder
- scanner := bufio.NewScanner(conn)
+ // The reader ends at the cap; N, the bytes it has left, is 0 once the
+ // answer has reached it.
+ answer := &io.LimitedReader{R: conn, N: maxAnswerSize}
+
+ scanner := bufio.NewScanner(answer)
for scanner.Scan() {
sb.WriteString(scanner.Text())
sb.WriteString("\n")
@@ -181,12 +198,18 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
return sb.String(), fmt.Errorf("read response: %w", err)
}
+ if answer.N == 0 {
+ return "", fmt.Errorf("read response: %w", errAnswerTooLarge)
+ }
+
return sb.String(), nil
}
// parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
-// The server is "" when the response has no referral.
-func (c *Client) parseIANAReferral(response string) (string, string) {
+// The server is "" when the response has no referral. A referral to a server
+// that is none of the five RIRs' is an error, so a forged or broken answer
+// cannot send the client to any other host or port.
+func (c *Client) parseIANAReferral(response string) (string, string, error) {
for line := range strings.SplitSeq(response, "\n") {
line = strings.TrimSpace(line)
@@ -197,27 +220,22 @@ func (c *Client) parseIANAReferral(response string) (string, string) {
switch {
case strings.Contains(server, "arin"):
- return RIRARIN, whoisServerARIN
+ return RIRARIN, whoisServerARIN, nil
case strings.Contains(server, "ripe"):
- return RIRRIPE, whoisServerRIPE
+ return RIRRIPE, whoisServerRIPE, nil
case strings.Contains(server, "apnic"):
- return RIRAPNIC, whoisServerAPNIC
+ return RIRAPNIC, whoisServerAPNIC, nil
case strings.Contains(server, "lacnic"):
- return RIRLACNIC, whoisServerLACNIC
+ return RIRLACNIC, whoisServerLACNIC, nil
case strings.Contains(server, "afrinic"):
- return RIRAFRNIC, whoisServerAFRINIC
+ return RIRAFRNIC, whoisServerAFRINIC, nil
default:
- // Unknown server, add port if missing
- if !strings.Contains(server, ":") {
- server += ":43"
- }
-
- return "", server
+ return "", "", fmt.Errorf("%w: %q", errUnknownServer, server)
}
}
}
- return "", ""
+ return "", "", nil
}
// parseResponse extracts ASN information from a WHOIS response.
diff --git a/internal/whois/whois_test.go b/internal/whois/whois_test.go
index 8d3a150..bcc8ef9 100644
--- a/internal/whois/whois_test.go
+++ b/internal/whois/whois_test.go
@@ -6,6 +6,7 @@ import (
"errors"
"io"
"net"
+ "strings"
"sync"
"syscall"
"testing"
@@ -71,6 +72,13 @@ const ianaNoObject = `% IANA WHOIS server
% This query returned 0 objects.
`
+// fillerLine is one of the lines a server that does not stop keeps sending.
+const fillerLine = "remarks: and more\n"
+
+// endlessAnswerSize is how many bytes of filler lines that server sends:
+// four times the client's cap on an answer.
+const endlessAnswerSize = 4 << 20
+
// fakeServers returns a DialContext for a whois.Client that connects to a
// fake server for each address in answers, which reads the query and sends
// answers[address]. Connecting to any other address is refused.
@@ -187,3 +195,52 @@ func TestLookupASNFailsWhenIANAAnswersNoBlock(t *testing.T) {
err)
}
}
+
+// TestLookupASNFailsWhenTheRIRAnswerDoesNotEnd looks up an AS whose RIR
+// server follows the record with filler lines far past any real answer.
+// The client stops reading at its cap and fails instead of keeping them.
+func TestLookupASNFailsWhenTheRIRAnswerDoesNotEnd(t *testing.T) {
+ t.Parallel()
+
+ endless := ripeRecord +
+ strings.Repeat(fillerLine, endlessAnswerSize/len(fillerLine))
+
+ _, err := lookUp(t, map[string]string{
+ ianaServer: ianaReferral,
+ ripeServer: endless,
+ })
+ if err == nil || errors.Is(err, whois.ErrNoReferral) {
+ t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral",
+ err)
+ }
+}
+
+// TestLookupASNFailsOnAReferralToAnotherServer looks up an AS when IANA's
+// answer refers it to a server none of the five RIRs run, with a port and
+// without. The client follows a referral only to an RIR's server, so it
+// fails instead of asking that server, which would answer with a record.
+func TestLookupASNFailsOnAReferralToAnotherServer(t *testing.T) {
+ t.Parallel()
+
+ // Each case maps the server on the "refer:" line to the address the
+ // client would dial to follow it. IANA's answer also names the AS's
+ // block, so a client that dropped the referral would report no RIR.
+ for refer, address := range map[string]string{
+ "whois.example.net:8080": "whois.example.net:8080",
+ "whois.example.net": "whois.example.net:43",
+ } {
+ t.Run(refer, func(t *testing.T) {
+ t.Parallel()
+
+ _, err := lookUp(t, map[string]string{
+ ianaServer: "refer: " + refer + "\n\n" +
+ "as-block: 3154-3353\n",
+ address: ripeRecord,
+ })
+ if err == nil || errors.Is(err, whois.ErrNoReferral) {
+ t.Errorf("LookupASN error = %v, want one that is not "+
+ "ErrNoReferral", err)
+ }
+ })
+ }
+}
--
2.54.0
From 28f60500cb3d28e877e45e96b60336b6c5fe91b8 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Tue, 20 Oct 2026 00:52:32 +0200
Subject: [PATCH 27/45] Read the AS page from bounded lookups (closes #104)
The AS page and /api/v1/as/{asn} grouped every route the AS originates
by prefix, reading one table row per peer and prefix and sorting them
all. They now take the AS's prefixes one at a time: one search of a new
index on origin_asn and prefix, which replaces the origin_asn index and
is written exactly when it was, and one search of the prefix and
last_updated index for the time of the prefix's newest route. Neither
reads the table. For a prefix more than one AS originates, that time is
now the newest route's from any origin. A hand-run benchmark reads an
AS of 5,000 prefixes from 500 peers.
Model: opus-5-5
---
internal/database/database.go | 121 ++++++------
internal/database/database_internal_test.go | 196 ++++++++++++++++++++
internal/database/schema.sql | 9 +-
3 files changed, 265 insertions(+), 61 deletions(-)
diff --git a/internal/database/database.go b/internal/database/database.go
index 3d4e89e..279940d 100644
--- a/internal/database/database.go
+++ b/internal/database/database.go
@@ -1673,8 +1673,9 @@ func (d *Database) GetASDetails(asn int) (*ASN, []LiveRoute, error) {
return d.GetASDetailsContext(context.Background(), asn)
}
-// GetASDetailsContext returns detailed information about an AS including
-// prefixes with context support
+// GetASDetailsContext returns the AS and the prefixes it originates, the IPv4
+// ones first, each once with the time of its newest route. It makes two index
+// searches per prefix, however many peers carry the prefix.
func (d *Database) GetASDetailsContext(
ctx context.Context, asn int,
) (*ASN, []LiveRoute, error) {
@@ -1701,27 +1702,12 @@ func (d *Database) GetASDetailsContext(
asnInfo.Handle = handle.String
asnInfo.Description = description.String
- // Get prefixes announced by this AS from both tables
- queryV4 := `
- SELECT prefix, mask_length, MAX(last_updated) as last_updated
- FROM live_routes_v4
- WHERE origin_asn = ?
- GROUP BY prefix, mask_length
- `
-
- allPrefixes, err := d.originPrefixes(ctx, queryV4, asn, ipVersionV4)
+ allPrefixes, err := d.originPrefixes(ctx, asn, ipVersionV4)
if err != nil {
return &asnInfo, nil, err
}
- queryV6 := `
- SELECT prefix, mask_length, MAX(last_updated) as last_updated
- FROM live_routes_v6
- WHERE origin_asn = ?
- GROUP BY prefix, mask_length
- `
-
- prefixesV6, err := d.originPrefixes(ctx, queryV6, asn, ipVersionV6)
+ prefixesV6, err := d.originPrefixes(ctx, asn, ipVersionV6)
if err != nil {
return &asnInfo, allPrefixes, err
}
@@ -2452,66 +2438,87 @@ func (d *Database) scanRouteTimestamp(
}
}
-// originPrefixes runs query, which selects the prefix, mask length and latest
-// last_updated of every prefix asn originates in the live route table of
-// ipVersion. Rows that fail to scan or whose timestamp does not parse are
-// logged and skipped.
+// The AS page lists the prefixes an AS originates one at a time, each in two
+// index searches that read no table row: one of the origin_asn and prefix
+// index for the AS's first prefix after the previous one, and one of the
+// prefix and last_updated index for the time of its newest route. Neither
+// reads more for a prefix that more peers carry.
+const (
+ nextOriginPrefixV4SQL = `SELECT prefix FROM live_routes_v4
+ WHERE origin_asn = ? AND prefix > ? ORDER BY prefix LIMIT 1`
+ newestRouteTimeV4SQL = `SELECT last_updated FROM live_routes_v4
+ WHERE prefix = ? ORDER BY last_updated DESC LIMIT 1`
+
+ nextOriginPrefixV6SQL = `SELECT prefix FROM live_routes_v6
+ WHERE origin_asn = ? AND prefix > ? ORDER BY prefix LIMIT 1`
+ newestRouteTimeV6SQL = `SELECT last_updated FROM live_routes_v6
+ WHERE prefix = ? ORDER BY last_updated DESC LIMIT 1`
+)
+
+// originPrefixes returns every prefix asn originates in the live route table
+// of ipVersion, in text order, each with its mask length and the time of the
+// prefix's newest route, whichever AS that route names as origin. A prefix
+// whose routes were all withdrawn between the two searches is left out.
func (d *Database) originPrefixes(
- ctx context.Context, query string, asn, ipVersion int,
+ ctx context.Context, asn, ipVersion int,
) ([]LiveRoute, error) {
- family := ipv4Family
+ family, nextSQL, newestSQL := ipv4Family, nextOriginPrefixV4SQL,
+ newestRouteTimeV4SQL
if ipVersion == ipVersionV6 {
- family = ipv6Family
+ family, nextSQL, newestSQL = ipv6Family, nextOriginPrefixV6SQL,
+ newestRouteTimeV6SQL
}
- rows, err := d.db.QueryContext(ctx, query, asn)
+ nextStmt, err := d.db.PrepareContext(ctx, nextSQL)
if err != nil {
- return nil, fmt.Errorf("failed to query %s prefixes: %w", family, err)
+ return nil, fmt.Errorf("failed to prepare %s prefix query: %w",
+ family, err)
}
- defer func() { _ = rows.Close() }()
+ defer func() { _ = nextStmt.Close() }()
+
+ newestStmt, err := d.db.PrepareContext(ctx, newestSQL)
+ if err != nil {
+ return nil, fmt.Errorf("failed to prepare %s route time query: %w",
+ family, err)
+ }
+ defer func() { _ = newestStmt.Close() }()
var routes []LiveRoute
- for rows.Next() {
- var (
- route LiveRoute
- lastUpdatedStr string
- )
+ previous := "" // every prefix sorts after the empty string
+
+ for {
+ var prefix string
+
+ err := nextStmt.QueryRowContext(ctx, asn, previous).Scan(&prefix)
+ if errors.Is(err, sql.ErrNoRows) {
+ return routes, nil
+ }
- err := rows.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr)
if err != nil {
- d.logger.Error("Failed to scan "+family+" prefix row",
- "error", err, "asn", asn)
+ return nil, fmt.Errorf("failed to list %s prefixes: %w", family, err)
+ }
+ previous = prefix
+ route := LiveRoute{Prefix: prefix, IPVersion: ipVersion, OriginASN: asn}
+
+ err = newestStmt.QueryRowContext(ctx, prefix).Scan(&route.LastUpdated)
+ if errors.Is(err, sql.ErrNoRows) {
continue
}
- // Parse the timestamp string
- route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00",
- lastUpdatedStr)
if err != nil {
- // Try without timezone
- route.LastUpdated, err = time.Parse("2006-01-02 15:04:05",
- lastUpdatedStr)
- if err != nil {
- d.logger.Error("Failed to parse timestamp",
- "error", err, "timestamp", lastUpdatedStr)
+ return nil, fmt.Errorf("failed to query route time of %s: %w",
+ prefix, err)
+ }
- continue
- }
+ route.MaskLength, err = prefixMaskLength(prefix)
+ if err != nil {
+ return nil, err
}
- route.OriginASN = asn
- route.IPVersion = ipVersion
routes = append(routes, route)
}
-
- err = rows.Err()
- if err != nil {
- return nil, fmt.Errorf("failed to read %s prefixes: %w", family, err)
- }
-
- return routes, nil
}
// firstPrefixesByLength returns the first n prefixes, in text order, with the
diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go
index 7a48559..41d2c22 100644
--- a/internal/database/database_internal_test.go
+++ b/internal/database/database_internal_test.go
@@ -10,6 +10,7 @@ import (
"net/netip"
"slices"
"strconv"
+ "strings"
"sync"
"testing"
"time"
@@ -1214,6 +1215,138 @@ func TestLiveRouteUpdateSetsOriginWhenItChanges(t *testing.T) {
}
}
+// TestGetASDetailsListsEachPrefixOnce stores two IPv4 prefixes and an IPv6
+// prefix an AS originates, two of them from two peers at different times, and
+// a prefix another AS originates. The AS details list each of the AS's
+// prefixes once, IPv4 first and in text order, with its mask length and the
+// time of its newest route.
+func TestGetASDetailsListsEachPrefixOnce(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ const asn, otherASN = 64500, 64501
+
+ ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+ _, err = db.GetOrCreateASN(asn, ts)
+ if err != nil {
+ t.Fatalf("GetOrCreateASN: %v", err)
+ }
+
+ v4Older := mkV4Route(t, "192.0.2.0/24", asn, ts)
+ v4Newer := mkV4Route(t, "192.0.2.0/24", asn, ts.Add(time.Minute))
+ v4Newer.PeerIP = secondV4PeerIP
+ v4Single := mkV4Route(t, "198.51.100.128/25", asn, ts)
+ v6Newer := mkV6Route(testV6Prefix, asn, ts.Add(time.Minute))
+ v6Older := mkV6Route(testV6Prefix, asn, ts)
+ v6Older.PeerIP = secondV6PeerIP
+ other := mkV4Route(t, "203.0.113.0/24", otherASN, ts)
+
+ err = db.UpsertLiveRouteBatch([]*LiveRoute{
+ v4Older, v4Newer, v4Single, v6Newer, v6Older, other,
+ })
+ if err != nil {
+ t.Fatalf("UpsertLiveRouteBatch: %v", err)
+ }
+
+ _, prefixes, err := db.GetASDetails(asn)
+ if err != nil {
+ t.Fatalf("GetASDetails: %v", err)
+ }
+
+ want := []*LiveRoute{v4Newer, v4Single, v6Newer}
+
+ listedOnce := slices.EqualFunc(prefixes, want,
+ func(got LiveRoute, want *LiveRoute) bool {
+ return got.Prefix == want.Prefix &&
+ got.MaskLength == want.MaskLength &&
+ got.IPVersion == want.IPVersion &&
+ got.OriginASN == asn &&
+ got.LastUpdated.Equal(want.LastUpdated)
+ })
+ if !listedOnce {
+ t.Errorf("GetASDetails listed %+v, want the prefixes and times of %+v",
+ prefixes, want)
+ }
+}
+
+// TestASPrefixQueriesUseCoveringIndexes checks the plan of each query that
+// lists the prefixes an AS originates: a search of a covering index and no
+// temporary B-tree, so the list reads one index entry per prefix, not one
+// table row per route. Grouping the AS's routes by prefix read every route's
+// row and sorted them in a temporary B-tree
+// (https://git.eeqj.de/sneak/routewatch/issues/104).
+func TestASPrefixQueriesUseCoveringIndexes(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ tests := []struct {
+ query string
+ args []any
+ index string
+ }{
+ {nextOriginPrefixV4SQL, []any{0, ""}, "idx_live_routes_v4_origin_prefix"},
+ {newestRouteTimeV4SQL, []any{""}, "idx_live_routes_v4_prefix_last_updated"},
+ {nextOriginPrefixV6SQL, []any{0, ""}, "idx_live_routes_v6_origin_prefix"},
+ {newestRouteTimeV6SQL, []any{""}, "idx_live_routes_v6_prefix_last_updated"},
+ }
+
+ for _, tt := range tests {
+ plan := queryPlan(t, db, tt.query, tt.args...)
+
+ if !strings.Contains(plan, "USING COVERING INDEX "+tt.index+" ") ||
+ strings.Contains(plan, "TEMP B-TREE") {
+ t.Errorf("plan of %s:\n%s\nwant a search of covering index %s "+
+ "and no temporary B-tree", tt.query, plan, tt.index)
+ }
+ }
+}
+
+// queryPlan returns the steps of EXPLAIN QUERY PLAN for query, one per line.
+func queryPlan(t *testing.T, db *Database, query string, args ...any) string {
+ t.Helper()
+
+ rows, err := db.db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query,
+ args...)
+ if err != nil {
+ t.Fatalf("EXPLAIN QUERY PLAN %s: %v", query, err)
+ }
+ defer func() { _ = rows.Close() }()
+
+ var steps []string
+
+ for rows.Next() {
+ var (
+ id, parent, unused int
+ detail string
+ )
+
+ err := rows.Scan(&id, &parent, &unused, &detail)
+ if err != nil {
+ t.Fatalf("reading the plan of %s: %v", query, err)
+ }
+
+ steps = append(steps, detail)
+ }
+
+ err = rows.Err()
+ if err != nil {
+ t.Fatalf("reading the plan of %s: %v", query, err)
+ }
+
+ return strings.Join(steps, "\n")
+}
+
// TestUpdatePrefixesBatchKeepsFirstSeen writes a prefix, then a batch with it
// and a new prefix. Only the new prefix is inserted and counted, and the known
// one keeps its first_seen time.
@@ -1518,3 +1651,66 @@ func writeBenchBatch(
b.Fatalf("UpdatePrefixesBatch: %v", err)
}
}
+
+// Parameters for BenchmarkGetASDetailsWithFiveHundredPeers: the AS originates
+// benchASPrefixes prefixes and every one of benchASPeers peers has a route for
+// each, 2.5 million routes in all.
+const (
+ benchASPrefixes = 5000
+ benchASPeers = 500
+)
+
+// BenchmarkGetASDetailsWithFiveHundredPeers times the read behind the AS page
+// for an AS that originates 5,000 IPv4 prefixes, each from 500 peers. It
+// builds that database first, one peer at a time, which takes some minutes and
+// about a GB of disk under TMPDIR. It is run by hand, not by make test, for
+// example in the test image:
+//
+// go test -run '^$' -bench GetASDetails -benchtime 10x ./internal/database/
+func BenchmarkGetASDetailsWithFiveHundredPeers(b *testing.B) {
+ db, err := New(&config.Config{StateDir: b.TempDir()}, logger.New())
+ if err != nil {
+ b.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+ _, err = db.GetOrCreateASN(benchFirstASN, ts)
+ if err != nil {
+ b.Fatalf("GetOrCreateASN: %v", err)
+ }
+
+ for peer := range benchASPeers {
+ peerIP := fmt.Sprintf("198.18.%d.%d", peer/256, peer%256)
+
+ routes := make([]*LiveRoute, 0, benchASPrefixes)
+ for prefix := range benchASPrefixes {
+ routes = append(routes, &LiveRoute{
+ Prefix: fmt.Sprintf("%d.%d.%d.0/%d",
+ 1+prefix>>16, (prefix>>8)%256, prefix%256, benchMaskLength),
+ MaskLength: benchMaskLength,
+ IPVersion: ipVersionV4,
+ OriginASN: benchFirstASN,
+ PeerIP: peerIP,
+ ASPath: []int{benchFirstASN + 1 + peer, benchFirstASN},
+ NextHop: peerIP,
+ LastUpdated: ts,
+ })
+ }
+
+ writeBenchBatch(b, db, routes, nil, ts)
+ }
+
+ for b.Loop() {
+ _, prefixes, err := db.GetASDetails(benchFirstASN)
+ if err != nil {
+ b.Fatalf("GetASDetails: %v", err)
+ }
+
+ if len(prefixes) != benchASPrefixes {
+ b.Fatalf("GetASDetails listed %d prefixes, want %d",
+ len(prefixes), benchASPrefixes)
+ }
+ }
+}
diff --git a/internal/database/schema.sql b/internal/database/schema.sql
index 6bd65a6..5c0ca22 100644
--- a/internal/database/schema.sql
+++ b/internal/database/schema.sql
@@ -128,7 +128,7 @@ CREATE TABLE IF NOT EXISTS live_routes_v6 (
-- prefix, last_updated: a prefix's routes, for the prefix page and its API
-- answer, the IP lookup and its peer count, and the check whether a prefix
-- still has a live route; and a prefix's newest route for the prefix-length
--- pages, in one search whatever the number of peers.
+-- pages and the AS page, in one search whatever the number of peers.
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
-- mask_length: the prefix-length pages' random sample, the row id range of a
@@ -139,9 +139,10 @@ CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask
-- prefixes in text order, and the prefix distribution read at startup.
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
--- origin_asn: the prefixes an AS originates, for the AS page and its API answer.
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_asn ON live_routes_v6(origin_asn);
+-- origin_asn, prefix: the prefixes an AS originates, each in one search
+-- whatever the number of peers, for the AS page and its API answer.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_prefix ON live_routes_v4(origin_asn, prefix);
+CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_prefix ON live_routes_v6(origin_asn, prefix);
-- last_updated: the oldest and newest route times in /api/v1/stats and
-- /status.json.
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
--
2.54.0
From 76c2e523b5d7d5ff038ae6c09a39cf89d954a172 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Tue, 20 Oct 2026 01:06:51 +0200
Subject: [PATCH 28/45] Checkpoint without waiting on readers and log a busy
checkpoint (closes #102)
The 5-second checkpoint ran in TRUNCATE mode, which waits up to the
5-second busy timeout for every open read while holding the database
lock every batch write needs. It now runs in PASSIVE mode, which never
waits. A WAL file over 64 MiB instead gets one TRUNCATE checkpoint whose
wait for reads is capped at 500 ms and logged; it empties the file once
no read is open, and while reads follow one another with no pause it
can give up every time. A checkpoint that leaves part of the WAL
uncopied returns ErrCheckpointIncomplete, which the maintainer logs as
incomplete instead of completed. The writing connection's
journal_size_limit is 64 MiB.
Model: opus-5-5
---
internal/database/database.go | 118 ++++++++-
internal/database/database_internal_test.go | 249 ++++++++++++++++++
internal/routewatch/dbmaintainer.go | 15 +-
.../routewatch/dbmaintainer_internal_test.go | 54 ++++
4 files changed, 430 insertions(+), 6 deletions(-)
create mode 100644 internal/routewatch/dbmaintainer_internal_test.go
diff --git a/internal/database/database.go b/internal/database/database.go
index 279940d..16425e9 100644
--- a/internal/database/database.go
+++ b/internal/database/database.go
@@ -81,6 +81,23 @@ const (
sqliteHardHeapLimitBytes = 2147483648
)
+// Settings that bound the WAL file's size once no read is open (see
+// Checkpoint).
+const (
+ // sqliteWALSizeLimitBytes (64 MiB) is the WAL file size past which
+ // Checkpoint runs a TRUNCATE checkpoint. It is also the
+ // journal_size_limit of the connection kept for writing: when SQLite
+ // starts the WAL again from the beginning, it cuts a larger WAL file down
+ // to this size.
+ sqliteWALSizeLimitBytes = 67108864
+ // truncateBusyTimeoutMs is the busy timeout of the TRUNCATE checkpoint:
+ // the longest it waits for open reads, and so the longest it keeps batch
+ // writes waiting beyond the copy itself.
+ truncateBusyTimeoutMs = 500
+ // walFileSuffix ends the name of the WAL file, next to the database file.
+ walFileSuffix = "-wal"
+)
+
// Settings of the ANALYZE that Analyze runs.
const (
// analysisLimit is about how many rows of each index ANALYZE reads
@@ -112,6 +129,12 @@ var (
ErrNoRoute = errors.New("no route found")
// ErrNoStaleASN is returned when no ASN needs WHOIS refresh
ErrNoStaleASN = errors.New("no stale ASN found")
+ // ErrCheckpointIncomplete is returned by Checkpoint when it left part of
+ // the WAL uncopied, or left a WAL file over sqliteWALSizeLimitBytes
+ // untruncated, because reads still needed the WAL or SQLite reported the
+ // checkpoint busy. A later checkpoint that runs when no read is open
+ // finishes the work.
+ ErrCheckpointIncomplete = errors.New("WAL checkpoint incomplete")
errInvalidASN = errors.New("invalid ASN")
errSameASNInPeering = errors.New("cannot create peering with same ASN")
@@ -194,6 +217,13 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
err)
}
+ _, err = writer.ExecContext(ctx,
+ fmt.Sprintf("PRAGMA journal_size_limit=%d", sqliteWALSizeLimitBytes))
+ if err != nil {
+ return nil, fmt.Errorf(
+ "failed to set the writing connection's WAL size limit: %w", err)
+ }
+
database := &Database{
db: db, writer: writer, logger: logger, path: dbPath,
counts: &liveCounts{},
@@ -2241,9 +2271,24 @@ func (d *Database) Analyze(ctx context.Context) error {
return nil
}
-// Checkpoint runs a WAL checkpoint to transfer data from the WAL to the main database.
-// Uses TRUNCATE mode which blocks writers briefly but ensures complete checkpoint,
-// keeping the WAL small for fast read performance. It holds the database lock.
+// Checkpoint copies the WAL into the database file. It holds the database
+// lock, so batch writes wait while it runs.
+//
+// It runs a PASSIVE checkpoint, which never waits: it copies what no open read
+// still needs, and leaves the file as large as it is. When the file is over
+// sqliteWALSizeLimitBytes, Checkpoint runs one TRUNCATE checkpoint instead,
+// which waits at most truncateBusyTimeoutMs for the open reads to end, then
+// copies all of the WAL and empties the file, or gives up. It is logged with
+// how long it took and whether it emptied the file.
+//
+// So a file over the limit is emptied by the first Checkpoint that finds no
+// read open, at its start or at some moment of its wait. While reads follow
+// one another with no pause between them, every TRUNCATE checkpoint can give
+// up after its full wait, and the file can keep growing for as long as they
+// do.
+//
+// It returns ErrCheckpointIncomplete when it leaves part of the WAL uncopied,
+// or a file over the limit not emptied.
func (d *Database) Checkpoint(ctx context.Context) error {
d.lock("Checkpoint")
defer d.unlock()
@@ -2253,11 +2298,32 @@ func (d *Database) Checkpoint(ctx context.Context) error {
return fmt.Errorf("failed to checkpoint WAL: %w", err)
}
- _, err = d.writer.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
+ wal, err := os.Stat(d.path + walFileSuffix)
+ if err != nil {
+ return fmt.Errorf("failed to read the WAL file's size: %w", err)
+ }
+
+ if wal.Size() > sqliteWALSizeLimitBytes {
+ return d.truncateWAL(ctx, wal.Size())
+ }
+
+ var busy, walPages, copiedPages int
+
+ err = d.writer.QueryRowContext(ctx, "PRAGMA wal_checkpoint(PASSIVE)").
+ Scan(&busy, &walPages, &copiedPages)
if err != nil {
return fmt.Errorf("failed to checkpoint WAL: %w", err)
}
+ if busy != 0 {
+ return fmt.Errorf("%w: SQLite reported it busy", ErrCheckpointIncomplete)
+ }
+
+ if copiedPages < walPages {
+ return fmt.Errorf("%w: copied %d of %d WAL pages",
+ ErrCheckpointIncomplete, copiedPages, walPages)
+ }
+
return nil
}
@@ -2273,6 +2339,50 @@ func (d *Database) Ping(ctx context.Context) error {
return nil
}
+// truncateWAL runs the TRUNCATE checkpoint of Checkpoint on a WAL file of
+// walBytes, with the writing connection's busy timeout lowered to
+// truncateBusyTimeoutMs for it, and logs how long it took. The caller holds the
+// database lock.
+func (d *Database) truncateWAL(ctx context.Context, walBytes int64) error {
+ _, err := d.writer.ExecContext(ctx,
+ fmt.Sprintf("PRAGMA busy_timeout=%d", truncateBusyTimeoutMs))
+ if err != nil {
+ return fmt.Errorf("failed to lower the busy timeout to truncate WAL: %w",
+ err)
+ }
+
+ started := time.Now()
+
+ var busy, walPages, copiedPages int
+
+ checkpointErr := d.writer.QueryRowContext(ctx,
+ "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walPages, &copiedPages)
+ took := time.Since(started)
+
+ // Restored even when ctx has ended, so that no batch write runs with the
+ // short timeout.
+ _, restoreErr := d.writer.ExecContext(context.WithoutCancel(ctx),
+ fmt.Sprintf("PRAGMA busy_timeout=%d", sqliteBusyTimeoutMs))
+ if checkpointErr != nil || restoreErr != nil {
+ return fmt.Errorf("failed to truncate WAL: %w",
+ errors.Join(checkpointErr, restoreErr))
+ }
+
+ if busy != 0 {
+ d.logger.Warn("WAL file over its size limit not truncated",
+ "wal_bytes", walBytes, "wal_pages", walPages,
+ "copied_pages", copiedPages, "duration", took)
+
+ return fmt.Errorf("%w: WAL file of %d bytes not truncated",
+ ErrCheckpointIncomplete, walBytes)
+ }
+
+ d.logger.Info("WAL file over its size limit truncated",
+ "wal_bytes", walBytes, "duration", took)
+
+ return nil
+}
+
// The debug messages lock logs, with the operation's name, when it starts to
// wait for the database mutex and when it holds it.
const (
diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go
index 41d2c22..cd82917 100644
--- a/internal/database/database_internal_test.go
+++ b/internal/database/database_internal_test.go
@@ -8,6 +8,7 @@ import (
"log/slog"
"math/rand/v2"
"net/netip"
+ "os"
"slices"
"strconv"
"strings"
@@ -68,6 +69,15 @@ const (
lockWaitTimeout = time.Millisecond
// lockTestASN is the AS the batch write in the lock tests stores.
lockTestASN = 64500
+ // readerCheckpointLimit is how long a checkpoint, and a batch write
+ // waiting for it, may take while a reader is open: well under the busy
+ // timeout, which a checkpoint that waits for the reader takes.
+ readerCheckpointLimit = time.Second
+ // truncateCheckpointLimit is how long a checkpoint of a WAL file over
+ // sqliteWALSizeLimitBytes may take to give up while a reader is open: its
+ // wait for the reader plus a second, well under the busy timeout.
+ truncateCheckpointLimit = truncateBusyTimeoutMs*time.Millisecond +
+ time.Second
)
// Parameters for the prefix-length sample tests.
@@ -458,6 +468,33 @@ func TestWritingConnectionCache(t *testing.T) {
}
}
+// TestWritingConnectionWALSizeLimit checks that the connection kept for
+// writing, the one that writes the WAL, has the WAL size limit, which cuts the
+// WAL file back when SQLite starts the WAL again from the beginning.
+func TestWritingConnectionWALSizeLimit(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ var limit int64
+
+ err = db.writer.QueryRowContext(t.Context(), "PRAGMA journal_size_limit").
+ Scan(&limit)
+ if err != nil {
+ t.Fatalf("failed to read the writing connection's journal_size_limit: %v",
+ err)
+ }
+
+ if limit != sqliteWALSizeLimitBytes {
+ t.Errorf("writing connection: journal_size_limit = %d, want %d",
+ limit, sqliteWALSizeLimitBytes)
+ }
+}
+
// readPragma reads the integer value of PRAGMA name on connection i, c.
func readPragma(
ctx context.Context, t *testing.T, c *sql.Conn, i int, name string,
@@ -800,6 +837,218 @@ func TestCheckpointLockWaitEndsWithContext(t *testing.T) {
db.mu.Unlock()
}
+// TestCheckpointDoesNotWaitForReader keeps a read transaction on
+// live_routes_v4 open while a route is committed, then starts Checkpoint and a
+// batch write together. It checks that both end within readerCheckpointLimit,
+// whichever of them takes the database lock first, and that Checkpoint reports
+// the part of the WAL the reader still needs as left uncopied. A TRUNCATE
+// checkpoint waited the busy timeout for the reader while holding the lock
+// (https://git.eeqj.de/sneak/routewatch/issues/102).
+func TestCheckpointDoesNotWaitForReader(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ ctx := context.Background()
+ ts := time.Now().UTC()
+
+ err = db.UpsertLiveRouteBatch(
+ []*LiveRoute{mkV4Route(t, "198.51.100.0/24", lockTestASN, ts)})
+ if err != nil {
+ t.Fatalf("failed to store the route the reader reads: %v", err)
+ }
+
+ reader, err := db.db.Conn(ctx)
+ if err != nil {
+ t.Fatalf("failed to open the reading connection: %v", err)
+ }
+ defer func() { _ = reader.Close() }()
+
+ // A plain BEGIN starts a transaction that only reads, unlike the BEGIN
+ // IMMEDIATE of _txlock. It keeps the snapshot its first read takes until
+ // it ends.
+ _, err = reader.ExecContext(ctx, "BEGIN")
+ if err != nil {
+ t.Fatalf("failed to begin the read transaction: %v", err)
+ }
+ defer func() { _, _ = reader.ExecContext(ctx, "ROLLBACK") }()
+
+ var routes int
+
+ err = reader.QueryRowContext(ctx,
+ "SELECT count(*) FROM live_routes_v4").Scan(&routes)
+ if err != nil {
+ t.Fatalf("failed to read the live routes: %v", err)
+ }
+
+ err = db.UpsertLiveRouteBatch(
+ []*LiveRoute{mkV4Route(t, "203.0.113.0/24", lockTestASN, ts)})
+ if err != nil {
+ t.Fatalf("failed to store a route after the read: %v", err)
+ }
+
+ started := time.Now()
+ checkpointed := make(chan error, 1)
+
+ go func() { checkpointed <- db.Checkpoint(ctx) }()
+
+ // A route write, not an AS write: the first new AS loads the embedded AS
+ // names, which takes seconds under -race.
+ route := mkV4Route(t, "192.0.2.0/24", lockTestASN, ts)
+ written := make(chan error, 1)
+
+ go func() { written <- db.UpsertLiveRouteBatch([]*LiveRoute{route}) }()
+
+ err = <-checkpointed
+ if !errors.Is(err, ErrCheckpointIncomplete) {
+ t.Errorf("Checkpoint = %v with a reader open, want %v",
+ err, ErrCheckpointIncomplete)
+ }
+
+ took := time.Since(started)
+ if took >= readerCheckpointLimit {
+ t.Errorf("Checkpoint took %v with a reader open, want under %v",
+ took, readerCheckpointLimit)
+ }
+
+ err = <-written
+ if err != nil {
+ t.Errorf("batch write during Checkpoint: %v", err)
+ }
+
+ took = time.Since(started)
+ if took >= readerCheckpointLimit {
+ t.Errorf("batch write during Checkpoint ended after %v, want under %v",
+ took, readerCheckpointLimit)
+ }
+}
+
+// TestCheckpointEmptiesWALOnceNoReadIsOpen fills the WAL file past
+// sqliteWALSizeLimitBytes while the test holds a read open. With the read
+// open, it checks that Checkpoint gives up within truncateCheckpointLimit and
+// reports the checkpoint incomplete, and that a batch write still succeeds.
+// After the test closes the read, it checks that the next Checkpoint brings
+// the WAL file under the limit and gives the writing connection its busy
+// timeout back (https://git.eeqj.de/sneak/routewatch/issues/102).
+func TestCheckpointEmptiesWALOnceNoReadIsOpen(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ ctx := t.Context()
+
+ reader, err := db.db.Conn(ctx)
+ if err != nil {
+ t.Fatalf("failed to open the reading connection: %v", err)
+ }
+ defer func() { _ = reader.Close() }()
+
+ // A plain BEGIN starts a transaction that only reads, unlike the BEGIN
+ // IMMEDIATE of _txlock. It keeps the snapshot its first read takes until
+ // it ends, so no checkpoint can copy what is written after that read.
+ _, err = reader.ExecContext(ctx, "BEGIN")
+ if err != nil {
+ t.Fatalf("failed to begin the read transaction: %v", err)
+ }
+
+ var routes int
+
+ err = reader.QueryRowContext(ctx,
+ "SELECT count(*) FROM live_routes_v4").Scan(&routes)
+ if err != nil {
+ t.Fatalf("failed to read the live routes: %v", err)
+ }
+
+ fillWALPastLimit(ctx, t, db)
+
+ started := time.Now()
+
+ err = db.Checkpoint(ctx)
+ if !errors.Is(err, ErrCheckpointIncomplete) {
+ t.Errorf("Checkpoint = %v with a read open, want %v",
+ err, ErrCheckpointIncomplete)
+ }
+
+ took := time.Since(started)
+ if took >= truncateCheckpointLimit {
+ t.Errorf("Checkpoint took %v with a read open, want under %v",
+ took, truncateCheckpointLimit)
+ }
+
+ err = db.UpsertLiveRouteBatch([]*LiveRoute{
+ mkV4Route(t, "192.0.2.0/24", lockTestASN, time.Now().UTC()),
+ })
+ if err != nil {
+ t.Errorf("batch write after Checkpoint gave up: %v", err)
+ }
+
+ _, err = reader.ExecContext(ctx, "ROLLBACK")
+ if err != nil {
+ t.Fatalf("failed to end the read transaction: %v", err)
+ }
+
+ err = db.Checkpoint(ctx)
+ if err != nil {
+ t.Errorf("Checkpoint with no read open: %v", err)
+ }
+
+ size := walFileSize(t, db)
+ if size >= sqliteWALSizeLimitBytes {
+ t.Errorf("WAL file of %d bytes after Checkpoint with no read open, "+
+ "want under %d", size, sqliteWALSizeLimitBytes)
+ }
+
+ busyTimeout := readPragma(ctx, t, db.writer, 0, "busy_timeout")
+ if busyTimeout != sqliteBusyTimeoutMs {
+ t.Errorf("writing connection: busy_timeout = %d after the checkpoints, "+
+ "want %d", busyTimeout, sqliteBusyTimeoutMs)
+ }
+}
+
+// fillWALPastLimit writes sqliteWALSizeLimitBytes of random bytes to a table
+// of the test's own, which with SQLite's page and frame headers leaves the WAL
+// file of db over the limit, and checks that it does.
+func fillWALPastLimit(ctx context.Context, t *testing.T, db *Database) {
+ t.Helper()
+
+ _, err := db.writer.ExecContext(ctx, "CREATE TABLE wal_fill (data BLOB)")
+ if err != nil {
+ t.Fatalf("failed to create the table the test writes to: %v", err)
+ }
+
+ _, err = db.writer.ExecContext(ctx,
+ "INSERT INTO wal_fill VALUES (randomblob(?))", sqliteWALSizeLimitBytes)
+ if err != nil {
+ t.Fatalf("failed to fill the WAL: %v", err)
+ }
+
+ size := walFileSize(t, db)
+ if size <= sqliteWALSizeLimitBytes {
+ t.Fatalf("WAL file of %d bytes after the test's write, want over %d",
+ size, sqliteWALSizeLimitBytes)
+ }
+}
+
+// walFileSize returns the size of the WAL file of db.
+func walFileSize(t *testing.T, db *Database) int64 {
+ t.Helper()
+
+ wal, err := os.Stat(db.path + walFileSuffix)
+ if err != nil {
+ t.Fatalf("failed to read the WAL file's size: %v", err)
+ }
+
+ return wal.Size()
+}
+
// TestGetRandomPrefixesByLengthListsPrefixOnce stores one prefix of each
// family learned from two peers in the same second and checks that the
// prefix-length sample lists each of them once.
diff --git a/internal/routewatch/dbmaintainer.go b/internal/routewatch/dbmaintainer.go
index 8859a46..6ecdf0b 100644
--- a/internal/routewatch/dbmaintainer.go
+++ b/internal/routewatch/dbmaintainer.go
@@ -2,6 +2,7 @@ package routewatch
import (
"context"
+ "errors"
"log/slog"
"sync"
"time"
@@ -34,6 +35,12 @@ const (
analyzeTimeout = 5 * time.Minute
)
+// checkpointIncompleteMessage is logged, at debug level like a completed
+// checkpoint, when a checkpoint leaves part of the WAL for a later one: while
+// reads overlap, most do. The database warns when the WAL file stays over its
+// size limit.
+const checkpointIncompleteMessage = "WAL checkpoint incomplete"
+
// DBMaintainer handles background database maintenance tasks.
type DBMaintainer struct {
db database.Store
@@ -123,9 +130,13 @@ func (m *DBMaintainer) runCheckpoint(ctx context.Context) {
duration, err := runWithTimeout(ctx, checkpointTimeout, m.db.Checkpoint)
m.recordResult(&m.lastCheckpoint, &m.lastCheckpointError, &m.checkpointCount, err)
- if err != nil {
+ switch {
+ case errors.Is(err, database.ErrCheckpointIncomplete):
+ m.logger.Debug(checkpointIncompleteMessage,
+ "error", err, "duration", duration)
+ case err != nil:
m.logger.Error("WAL checkpoint failed", "error", err, "duration", duration)
- } else {
+ default:
m.logger.Debug("WAL checkpoint completed", "duration", duration)
}
}
diff --git a/internal/routewatch/dbmaintainer_internal_test.go b/internal/routewatch/dbmaintainer_internal_test.go
new file mode 100644
index 0000000..94d258a
--- /dev/null
+++ b/internal/routewatch/dbmaintainer_internal_test.go
@@ -0,0 +1,54 @@
+package routewatch
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "log/slog"
+ "testing"
+
+ "git.eeqj.de/sneak/routewatch/internal/database"
+)
+
+// incompleteCheckpointStore is a store whose every checkpoint leaves part of
+// the WAL uncopied.
+type incompleteCheckpointStore struct{ *mockStore }
+
+// Checkpoint reports the checkpoint incomplete.
+func (incompleteCheckpointStore) Checkpoint(context.Context) error {
+ return database.ErrCheckpointIncomplete
+}
+
+// TestIncompleteCheckpointNotLoggedAsCompleted runs a checkpoint that leaves
+// part of the WAL uncopied and checks that the maintainer logs it as
+// incomplete, not as completed, and does not count it as a checkpoint that
+// completed.
+func TestIncompleteCheckpointNotLoggedAsCompleted(t *testing.T) {
+ t.Parallel()
+
+ var logs bytes.Buffer
+
+ logger := slog.New(slog.NewJSONHandler(&logs,
+ &slog.HandlerOptions{Level: slog.LevelDebug}))
+ m := NewDBMaintainer(incompleteCheckpointStore{newMockStore()}, logger)
+
+ m.runCheckpoint(t.Context())
+
+ var logged struct {
+ Msg string `json:"msg"`
+ }
+
+ err := json.Unmarshal(logs.Bytes(), &logged)
+ if err != nil {
+ t.Fatalf("want one log message, got %q: %v", logs.String(), err)
+ }
+
+ if logged.Msg != checkpointIncompleteMessage {
+ t.Errorf("logged %q, want %q", logged.Msg, checkpointIncompleteMessage)
+ }
+
+ if m.checkpointCount != 0 {
+ t.Errorf("checkpointCount = %d after an incomplete checkpoint, want 0",
+ m.checkpointCount)
+ }
+}
--
2.54.0
From 89bdda18a52fdb78f27279f0f7ca93869bb7bba6 Mon Sep 17 00:00:00 2001
From: clawbot <35+clawbot@noreply.example.org>
Date: Tue, 20 Oct 2026 01:36:18 +0200
Subject: [PATCH 29/45] Add the HTTP hardening the policy requires before 1.0
(closes #108)
Every answer carries the six security headers REPO_POLICIES.md lists.
The Content-Security-Policy allows inline content only with a nonce made
for that answer; the pages put it on their style and script blocks and
drop their four style attributes.
The request log and /ip/ take a forwarded address only from a proxy
listed in TRUSTED_PROXIES, which stops the start when set but invalid:
the last X-Forwarded-For entry that is not a trusted proxy. X-Real-IP
and True-Client-IP are no longer read.
The JSON routes and the health check send fixed messages in place of
database, parser and resolver errors, and log the detail.
Model: opus-5-5
---
README.md | 36 +++-
internal/config/config.go | 50 ++++-
internal/config/config_internal_test.go | 51 +++++
internal/server/handlers.go | 90 ++++++---
internal/server/handlers_internal_test.go | 104 +++++++++-
internal/server/middleware.go | 112 ++++++++++-
internal/server/middleware_internal_test.go | 205 ++++++++++++++++++++
internal/server/routes.go | 3 +-
internal/server/server.go | 14 +-
internal/templates/as_detail.html | 2 +-
internal/templates/index.html | 15 +-
internal/templates/prefix_detail.html | 8 +-
internal/templates/prefix_length.html | 2 +-
internal/templates/status.html | 9 +-
14 files changed, 641 insertions(+), 60 deletions(-)
create mode 100644 internal/server/middleware_internal_test.go
diff --git a/README.md b/README.md
index 05ca2a5..800ec34 100644
--- a/README.md
+++ b/README.md
@@ -54,8 +54,9 @@ DEBUG=routewatch ./routewatch
picked at random, each with its newest route (HTML)
- `GET /prefixlength6/{length}` - The same for IPv6
- `GET /ip/{addr}` - The same JSON answer as `/api/v1/ip/{ip}`
-- `GET /ip/` - The same, for the caller's own address, or the one a proxy names
- in `True-Client-IP`, `X-Real-IP` or `X-Forwarded-For`
+- `GET /ip/` - The same, for the caller's own address: the address the request
+ came from or, when that is a trusted proxy (`TRUSTED_PROXIES`, see
+ Configuration), the client address in its `X-Forwarded-For`
### API v1
@@ -84,7 +85,17 @@ and 4 seconds for the database reads behind `/status.json` and `/api/v1/stats`.
An AS number, prefix or mask length that does not parse answers 400, as does a
mask length out of range for its IP version. An AS the database does not hold,
or a prefix with no live route, answers 404. The HTML pages give these errors as
-plain text.
+plain text. A 500 answer says only `Internal server error`, and a host name that
+does not resolve only `could not resolve host name`; the detail goes to the log.
+
+Every answer carries `Strict-Transport-Security`, `Content-Security-Policy`,
+`X-Frame-Options`, `X-Content-Type-Options`, `Referrer-Policy` and
+`Permissions-Policy`. The Content-Security-Policy lets a page load content only
+from the server, and does not allow inline content with `'unsafe-inline'`: the
+pages keep their CSS and JavaScript in inline `