Remove BGP keepalive logging and add peer tracking

- Created bgp_peers table to track all BGP peers
- Added PeerHandler to update peer last seen times for all message types
- Removed verbose BGP keepalive debug logging
- BGP keepalive messages now silently update peer tracking

Refactor HTML templates to use go:embed

- Created internal/templates package with embedded templates
- Moved status.html from inline const to separate file
- Templates are parsed once on startup
- Server now uses parsed template instead of raw string

Optimize AS data embedding with gzip compression

- Changed asinfo package to embed gzipped data (2.4MB vs 12MB)
- Updated Makefile to gzip AS data during update
- Added decompression during initialization
- Raw JSON file excluded from git
This commit is contained in:
Jeffrey Paul 2025-07-27 21:54:58 +02:00
parent ee80311ba1
commit 585ff63fae
18 changed files with 428 additions and 652241 deletions

3
.gitignore vendored
View File

@ -29,3 +29,6 @@ go.work.sum
# Temporary files
*.tmp
# Raw AS data (we only commit the gzipped version)
pkg/asinfo/asdata.json

View File

@ -25,5 +25,5 @@ run: build
asupdate:
@echo "Updating AS info data..."
@go run cmd/asinfo-gen/main.go > pkg/asinfo/asdata.json.tmp && \
mv pkg/asinfo/asdata.json.tmp pkg/asinfo/asdata.json
@go run cmd/asinfo-gen/main.go | gzip > pkg/asinfo/asdata.json.gz.tmp && \
mv pkg/asinfo/asdata.json.gz.tmp pkg/asinfo/asdata.json.gz

View File

@ -31,7 +31,9 @@ func main() {
if err != nil {
log.Fatalf("Failed to fetch CSV: %v", err)
}
defer resp.Body.Close()
defer func() {
_ = resp.Body.Close()
}()
if resp.StatusCode != http.StatusOK {
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
@ -59,17 +61,21 @@ func main() {
}
if err != nil {
log.Printf("Error reading CSV record: %v", err)
continue
}
if len(record) != 3 {
const expectedFields = 3
if len(record) != expectedFields {
log.Printf("Skipping invalid record: %v", record)
continue
}
asn, err := strconv.Atoi(record[0])
if err != nil {
log.Printf("Invalid ASN %q: %v", record[0], err)
continue
}

View File

@ -57,6 +57,16 @@ CREATE TABLE IF NOT EXISTS asn_peerings (
UNIQUE(from_asn_id, to_asn_id)
);
-- BGP peers that send us messages
CREATE TABLE IF NOT EXISTS bgp_peers (
id TEXT PRIMARY KEY,
peer_ip TEXT UNIQUE NOT NULL,
peer_asn INTEGER NOT NULL,
first_seen DATETIME NOT NULL,
last_seen DATETIME NOT NULL,
last_message_type TEXT
);
-- Live routing table: current state of announced routes
CREATE TABLE IF NOT EXISTS live_routes (
id TEXT PRIMARY KEY,
@ -91,6 +101,10 @@ CREATE INDEX IF NOT EXISTS idx_live_routes_origin
CREATE INDEX IF NOT EXISTS idx_live_routes_prefix
ON live_routes(prefix_id)
WHERE withdrawn_at IS NULL;
-- Indexes for bgp_peers table
CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
`
)
@ -468,6 +482,52 @@ func (d *Database) GetActiveLiveRoutes() ([]LiveRoute, error) {
return routes, rows.Err()
}
// UpdatePeer updates or creates a BGP peer record
func (d *Database) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error {
tx, err := d.db.Begin()
if err != nil {
return err
}
defer func() {
if err := tx.Rollback(); err != nil && err != sql.ErrTxDone {
d.logger.Error("Failed to rollback transaction", "error", err)
}
}()
var exists bool
err = tx.QueryRow("SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)", peerIP).Scan(&exists)
if err != nil {
return err
}
if exists {
_, err = tx.Exec(
"UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, last_message_type = ? WHERE peer_ip = ?",
peerASN, timestamp, messageType, peerIP,
)
} else {
_, err = tx.Exec(
"INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)",
generateUUID().String(), peerIP, peerASN, timestamp, timestamp, messageType,
)
}
if err != nil {
return err
}
if err = tx.Commit(); err != nil {
d.logger.Error("Failed to commit transaction for peer update",
"peer_ip", peerIP,
"peer_asn", peerASN,
"error", err,
)
return err
}
return nil
}
// GetStats returns database statistics
func (d *Database) GetStats() (Stats, error) {
var stats Stats

View File

@ -38,6 +38,9 @@ type Store interface {
// Statistics
GetStats() (Stats, error)
// Peer operations
UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error
// Lifecycle
Close() error
}

View File

@ -76,6 +76,10 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
dbHandler := NewDatabaseHandler(rw.db, rw.logger)
rw.streamer.RegisterHandler(dbHandler)
// Register peer tracking handler to track all peers
peerHandler := NewPeerHandler(rw.db, rw.logger)
rw.streamer.RegisterHandler(peerHandler)
// Start streaming
if err := rw.streamer.Start(); err != nil {
return err

View File

@ -158,6 +158,12 @@ func (m *mockStore) GetActiveLiveRoutes() ([]database.LiveRoute, error) {
return []database.LiveRoute{}, nil
}
// UpdatePeer mock implementation
func (m *mockStore) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error {
// Simple mock - just return nil
return nil
}
// Close mock implementation
func (m *mockStore) Close() error {
return nil

View File

@ -0,0 +1,49 @@
package routewatch
import (
"log/slog"
"strconv"
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/ristypes"
)
// PeerHandler tracks BGP peers from all message types
type PeerHandler struct {
db database.Store
logger *slog.Logger
}
// NewPeerHandler creates a new peer tracking handler
func NewPeerHandler(db database.Store, logger *slog.Logger) *PeerHandler {
return &PeerHandler{
db: db,
logger: logger,
}
}
// WantsMessage returns true for all message types since we track peers from all messages
func (h *PeerHandler) WantsMessage(_ string) bool {
return true
}
// HandleMessage processes a message to track peer information
func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
// Parse peer ASN from string
peerASN := 0
if msg.PeerASN != "" {
if asn, err := strconv.Atoi(msg.PeerASN); err == nil {
peerASN = asn
}
}
// Update peer in database
if err := h.db.UpdatePeer(msg.Peer, peerASN, msg.Type, msg.ParsedTimestamp); err != nil {
h.logger.Error("Failed to update peer",
"peer", msg.Peer,
"peer_asn", peerASN,
"message_type", msg.Type,
"error", err,
)
}
}

View File

@ -4,7 +4,6 @@ package server
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"os"
@ -12,6 +11,7 @@ import (
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/streamer"
"git.eeqj.de/sneak/routewatch/internal/templates"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
)
@ -226,191 +226,11 @@ func (s *Server) handleStats() http.HandlerFunc {
func (s *Server) handleStatusHTML() http.HandlerFunc {
return func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if _, err := fmt.Fprint(w, statusHTML); err != nil {
s.logger.Error("Failed to write HTML", "error", err)
}
}
}
const statusHTML = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>RouteWatch Status</title>
<style>
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
max-width: 1200px;
margin: 0 auto;
padding: 20px;
background: #f5f5f5;
tmpl := templates.StatusTemplate()
if err := tmpl.Execute(w, nil); err != nil {
s.logger.Error("Failed to render template", "error", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
h1 {
color: #333;
margin-bottom: 30px;
}
.status-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
gap: 20px;
margin-bottom: 30px;
}
.status-card {
background: white;
padding: 20px;
border-radius: 8px;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
}
.status-card h2 {
margin: 0 0 15px 0;
font-size: 18px;
color: #666;
}
.metric {
display: flex;
justify-content: space-between;
padding: 8px 0;
border-bottom: 1px solid #eee;
}
.metric:last-child {
border-bottom: none;
}
.metric-label {
color: #666;
}
.metric-value {
font-weight: 600;
color: #333;
}
.connected {
color: #22c55e;
}
.disconnected {
color: #ef4444;
}
.error {
background: #fee;
color: #c00;
padding: 10px;
border-radius: 4px;
margin-top: 20px;
}
</style>
</head>
<body>
<h1>RouteWatch Status</h1>
<div id="error" class="error" style="display: none;"></div>
<div class="status-grid">
<div class="status-card">
<h2>Connection Status</h2>
<div class="metric">
<span class="metric-label">Status</span>
<span class="metric-value" id="connected">-</span>
</div>
<div class="metric">
<span class="metric-label">Uptime</span>
<span class="metric-value" id="uptime">-</span>
</div>
</div>
<div class="status-card">
<h2>Stream Statistics</h2>
<div class="metric">
<span class="metric-label">Total Messages</span>
<span class="metric-value" id="total_messages">-</span>
</div>
<div class="metric">
<span class="metric-label">Messages/sec</span>
<span class="metric-value" id="messages_per_sec">-</span>
</div>
<div class="metric">
<span class="metric-label">Total Data</span>
<span class="metric-value" id="total_bytes">-</span>
</div>
<div class="metric">
<span class="metric-label">Throughput</span>
<span class="metric-value" id="mbits_per_sec">-</span>
</div>
</div>
<div class="status-card">
<h2>Database Statistics</h2>
<div class="metric">
<span class="metric-label">ASNs</span>
<span class="metric-value" id="asns">-</span>
</div>
<div class="metric">
<span class="metric-label">Total Prefixes</span>
<span class="metric-value" id="prefixes">-</span>
</div>
<div class="metric">
<span class="metric-label">IPv4 Prefixes</span>
<span class="metric-value" id="ipv4_prefixes">-</span>
</div>
<div class="metric">
<span class="metric-label">IPv6 Prefixes</span>
<span class="metric-value" id="ipv6_prefixes">-</span>
</div>
<div class="metric">
<span class="metric-label">Peerings</span>
<span class="metric-value" id="peerings">-</span>
</div>
<div class="metric">
<span class="metric-label">Live Routes</span>
<span class="metric-value" id="live_routes">-</span>
</div>
</div>
</div>
<script>
function formatBytes(bytes) {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
function formatNumber(num) {
return num.toLocaleString();
}
function updateStatus() {
fetch('/api/v1/stats')
.then(response => response.json())
.then(data => {
// Connection status
const connectedEl = document.getElementById('connected');
connectedEl.textContent = data.connected ? 'Connected' : 'Disconnected';
connectedEl.className = 'metric-value ' + (data.connected ? 'connected' : 'disconnected');
// Update all metrics
document.getElementById('uptime').textContent = data.uptime;
document.getElementById('total_messages').textContent = formatNumber(data.total_messages);
document.getElementById('messages_per_sec').textContent = data.messages_per_sec.toFixed(1);
document.getElementById('total_bytes').textContent = formatBytes(data.total_bytes);
document.getElementById('mbits_per_sec').textContent = data.mbits_per_sec.toFixed(2) + ' Mbps';
document.getElementById('asns').textContent = formatNumber(data.asns);
document.getElementById('prefixes').textContent = formatNumber(data.prefixes);
document.getElementById('ipv4_prefixes').textContent = formatNumber(data.ipv4_prefixes);
document.getElementById('ipv6_prefixes').textContent = formatNumber(data.ipv6_prefixes);
document.getElementById('peerings').textContent = formatNumber(data.peerings);
document.getElementById('live_routes').textContent = formatNumber(data.live_routes);
// Clear any errors
document.getElementById('error').style.display = 'none';
})
.catch(error => {
document.getElementById('error').textContent = 'Error fetching status: ' + error;
document.getElementById('error').style.display = 'block';
});
}
// Update immediately and then every 500ms
updateStatus();
setInterval(updateStatus, 500);
</script>
</body>
</html>
`

View File

@ -255,11 +255,7 @@ func (s *Streamer) stream(ctx context.Context) error {
case "RIS_PEER_STATE":
// RIS peer state messages - silently ignore
case "KEEPALIVE":
// BGP keepalive messages - just log at debug level
s.logger.Debug("BGP keepalive",
"peer", msg.Peer,
"peer_asn", msg.PeerASN,
)
// BGP keepalive messages - silently process
case "OPEN":
// BGP open messages
s.logger.Info("BGP session opened",

View File

@ -0,0 +1,181 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>RouteWatch Status</title>
<style>
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
max-width: 1200px;
margin: 0 auto;
padding: 20px;
background: #f5f5f5;
}
h1 {
color: #333;
margin-bottom: 30px;
}
.status-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
gap: 20px;
margin-bottom: 30px;
}
.status-card {
background: white;
padding: 20px;
border-radius: 8px;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
}
.status-card h2 {
margin: 0 0 15px 0;
font-size: 18px;
color: #666;
}
.metric {
display: flex;
justify-content: space-between;
padding: 8px 0;
border-bottom: 1px solid #eee;
}
.metric:last-child {
border-bottom: none;
}
.metric-label {
color: #666;
}
.metric-value {
font-weight: 600;
color: #333;
}
.connected {
color: #22c55e;
}
.disconnected {
color: #ef4444;
}
.error {
background: #fee;
color: #c00;
padding: 10px;
border-radius: 4px;
margin-top: 20px;
}
</style>
</head>
<body>
<h1>RouteWatch Status</h1>
<div id="error" class="error" style="display: none;"></div>
<div class="status-grid">
<div class="status-card">
<h2>Connection Status</h2>
<div class="metric">
<span class="metric-label">Status</span>
<span class="metric-value" id="connected">-</span>
</div>
<div class="metric">
<span class="metric-label">Uptime</span>
<span class="metric-value" id="uptime">-</span>
</div>
</div>
<div class="status-card">
<h2>Stream Statistics</h2>
<div class="metric">
<span class="metric-label">Total Messages</span>
<span class="metric-value" id="total_messages">-</span>
</div>
<div class="metric">
<span class="metric-label">Messages/sec</span>
<span class="metric-value" id="messages_per_sec">-</span>
</div>
<div class="metric">
<span class="metric-label">Total Data</span>
<span class="metric-value" id="total_bytes">-</span>
</div>
<div class="metric">
<span class="metric-label">Throughput</span>
<span class="metric-value" id="mbits_per_sec">-</span>
</div>
</div>
<div class="status-card">
<h2>Database Statistics</h2>
<div class="metric">
<span class="metric-label">ASNs</span>
<span class="metric-value" id="asns">-</span>
</div>
<div class="metric">
<span class="metric-label">Total Prefixes</span>
<span class="metric-value" id="prefixes">-</span>
</div>
<div class="metric">
<span class="metric-label">IPv4 Prefixes</span>
<span class="metric-value" id="ipv4_prefixes">-</span>
</div>
<div class="metric">
<span class="metric-label">IPv6 Prefixes</span>
<span class="metric-value" id="ipv6_prefixes">-</span>
</div>
<div class="metric">
<span class="metric-label">Peerings</span>
<span class="metric-value" id="peerings">-</span>
</div>
<div class="metric">
<span class="metric-label">Live Routes</span>
<span class="metric-value" id="live_routes">-</span>
</div>
</div>
</div>
<script>
function formatBytes(bytes) {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
function formatNumber(num) {
return num.toLocaleString();
}
function updateStatus() {
fetch('/api/v1/stats')
.then(response => response.json())
.then(data => {
// Connection status
const connectedEl = document.getElementById('connected');
connectedEl.textContent = data.connected ? 'Connected' : 'Disconnected';
connectedEl.className = 'metric-value ' + (data.connected ? 'connected' : 'disconnected');
// Update all metrics
document.getElementById('uptime').textContent = data.uptime;
document.getElementById('total_messages').textContent = formatNumber(data.total_messages);
document.getElementById('messages_per_sec').textContent = data.messages_per_sec.toFixed(1);
document.getElementById('total_bytes').textContent = formatBytes(data.total_bytes);
document.getElementById('mbits_per_sec').textContent = data.mbits_per_sec.toFixed(2) + ' Mbps';
document.getElementById('asns').textContent = formatNumber(data.asns);
document.getElementById('prefixes').textContent = formatNumber(data.prefixes);
document.getElementById('ipv4_prefixes').textContent = formatNumber(data.ipv4_prefixes);
document.getElementById('ipv6_prefixes').textContent = formatNumber(data.ipv6_prefixes);
document.getElementById('peerings').textContent = formatNumber(data.peerings);
document.getElementById('live_routes').textContent = formatNumber(data.live_routes);
// Clear any errors
document.getElementById('error').style.display = 'none';
})
.catch(error => {
document.getElementById('error').textContent = 'Error fetching status: ' + error;
document.getElementById('error').style.display = 'block';
});
}
// Update immediately and then every 500ms
updateStatus();
setInterval(updateStatus, 500);
</script>
</body>
</html>

View File

@ -0,0 +1,48 @@
// Package templates provides embedded HTML templates for the RouteWatch application
package templates
import (
_ "embed"
"html/template"
"sync"
)
//go:embed status.html
var statusHTML string
// Templates contains all parsed templates
type Templates struct {
Status *template.Template
}
var (
//nolint:gochecknoglobals // Singleton pattern for templates
defaultTemplates *Templates
//nolint:gochecknoglobals // Singleton pattern for templates
once sync.Once
)
// initTemplates parses all embedded templates
func initTemplates() {
var err error
defaultTemplates = &Templates{}
// Parse status template
defaultTemplates.Status, err = template.New("status").Parse(statusHTML)
if err != nil {
panic("failed to parse status template: " + err.Error())
}
}
// Get returns the singleton Templates instance
func Get() *Templates {
once.Do(initTemplates)
return defaultTemplates
}
// StatusTemplate returns the parsed status template
func StatusTemplate() *template.Template {
return Get().Status
}

File diff suppressed because it is too large Load Diff

BIN
pkg/asinfo/asdata.json.gz Normal file

Binary file not shown.

View File

@ -2,13 +2,16 @@
package asinfo
import (
"bytes"
"compress/gzip"
_ "embed"
"encoding/json"
"io"
"sync"
)
//go:embed asdata.json
var asDataJSON []byte
//go:embed asdata.json.gz
var asDataGZ []byte
// Info represents information about an Autonomous System
type Info struct {
@ -24,7 +27,9 @@ type Registry struct {
}
var (
//nolint:gochecknoglobals // Singleton pattern for embedded data
defaultRegistry *Registry
//nolint:gochecknoglobals // Singleton pattern for embedded data
once sync.Once
)
@ -34,8 +39,22 @@ func initRegistry() {
byASN: make(map[int]*Info),
}
// Decompress the gzipped data
gzReader, err := gzip.NewReader(bytes.NewReader(asDataGZ))
if err != nil {
panic("failed to create gzip reader: " + err.Error())
}
defer func() {
_ = gzReader.Close()
}()
jsonData, err := io.ReadAll(gzReader)
if err != nil {
panic("failed to decompress AS data: " + err.Error())
}
var asInfos []Info
if err := json.Unmarshal(asDataJSON, &asInfos); err != nil {
if err := json.Unmarshal(jsonData, &asInfos); err != nil {
panic("failed to unmarshal embedded AS data: " + err.Error())
}
@ -58,8 +77,9 @@ func Get(asn int) (*Info, bool) {
}
// Return a copy to prevent mutation
copy := *info
return &copy, true
infoCopy := *info
return &infoCopy, true
}
// GetDescription returns just the description for an ASN
@ -68,6 +88,7 @@ func GetDescription(asn int) string {
if !ok {
return ""
}
return info.Description
}
@ -77,6 +98,7 @@ func GetHandle(asn int) string {
if !ok {
return ""
}
return info.Handle
}
@ -135,5 +157,6 @@ func containsImpl(s, substr string) bool {
return true
}
}
return false
}