Add debug logging and optimize SQLite performance

- Add debug logging for goroutines and memory usage (enabled via DEBUG=routewatch)
- Increase SQLite connection pool from 1 to 10 connections for better concurrency
- Optimize SQLite pragmas for balanced performance and safety
- Add proper shutdown handling for peering handler
- Define constants to avoid magic numbers in code

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Jeffrey Paul 2025-07-28 15:45:06 +02:00
parent 9b649c98c9
commit 78d6e17c76
3 changed files with 66 additions and 16 deletions

View File

@ -30,6 +30,11 @@ const (
ipv4Offset = 12
ipv4Bits = 32
maxIPv4 = 0xFFFFFFFF
// Database connection pool settings
maxOpenConns = 10
maxIdleConns = 5
connMaxLifetime = 5 * time.Minute
)
// Common errors
@ -73,10 +78,10 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
}
// Set connection pool parameters
// Single connection to avoid locking issues with SQLite
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(1)
db.SetConnMaxLifetime(0)
// Allow multiple readers but single writer for SQLite WAL mode
db.SetMaxOpenConns(maxOpenConns)
db.SetMaxIdleConns(maxIdleConns)
db.SetConnMaxLifetime(connMaxLifetime)
database := &Database{db: db, logger: logger, path: dbPath}
@ -89,19 +94,20 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
// Initialize creates the database schema if it doesn't exist.
func (d *Database) Initialize() error {
// Set SQLite pragmas for better performance
// WARNING: These settings trade durability for speed
// Set SQLite pragmas for better performance with safety
pragmas := []string{
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
"PRAGMA synchronous=OFF", // Don't wait for disk writes - RISKY but FAST
"PRAGMA cache_size=-1048576", // 1GB cache (negative = KB)
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
"PRAGMA mmap_size=536870912", // 512MB memory-mapped I/O
"PRAGMA wal_autocheckpoint=10000", // Checkpoint every 10000 pages (less frequent)
"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
"PRAGMA page_size=8192", // Larger page size for better performance
"PRAGMA busy_timeout=30000", // 30 second busy timeout
"PRAGMA optimize", // Run optimizer
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
"PRAGMA synchronous=NORMAL", // Balance between safety and speed
"PRAGMA cache_size=-262144", // 256MB cache (negative = KB)
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
"PRAGMA mmap_size=268435456", // 256MB memory-mapped I/O
"PRAGMA wal_autocheckpoint=1000", // Checkpoint every 1000 pages
"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
"PRAGMA page_size=4096", // Standard page size
"PRAGMA busy_timeout=30000", // 30 second busy timeout
"PRAGMA optimize", // Run optimizer
"PRAGMA locking_mode=NORMAL", // Allow multiple connections
"PRAGMA read_uncommitted=true", // Allow dirty reads for better concurrency
}
for _, pragma := range pragmas {

View File

@ -139,6 +139,10 @@ func (rw *RouteWatch) Shutdown() {
rw.logger.Info("Flushing prefix handler")
rw.prefixHandler.Stop()
}
if rw.peeringHandler != nil {
rw.logger.Info("Flushing peering handler")
rw.peeringHandler.Stop()
}
// Stop services
rw.streamer.Stop()

View File

@ -4,6 +4,8 @@ import (
"context"
"os"
"os/signal"
"runtime"
"strings"
"syscall"
"time"
@ -14,8 +16,43 @@ import (
const (
// shutdownTimeout is the maximum time allowed for graceful shutdown
shutdownTimeout = 60 * time.Second
// debugInterval is how often to log debug stats
debugInterval = 60 * time.Second
// bytesPerMB is bytes per megabyte
bytesPerMB = 1024 * 1024
)
// logDebugStats logs goroutine count and memory usage
func logDebugStats(logger *logger.Logger) {
// Only run if DEBUG env var contains "routewatch"
debugEnv := os.Getenv("DEBUG")
if !strings.Contains(debugEnv, "routewatch") {
return
}
ticker := time.NewTicker(debugInterval)
defer ticker.Stop()
for range ticker.C {
var m runtime.MemStats
runtime.ReadMemStats(&m)
logger.Debug("System stats",
"goroutines", runtime.NumGoroutine(),
"alloc_mb", m.Alloc/bytesPerMB,
"total_alloc_mb", m.TotalAlloc/bytesPerMB,
"sys_mb", m.Sys/bytesPerMB,
"num_gc", m.NumGC,
"heap_alloc_mb", m.HeapAlloc/bytesPerMB,
"heap_sys_mb", m.HeapSys/bytesPerMB,
"heap_idle_mb", m.HeapIdle/bytesPerMB,
"heap_inuse_mb", m.HeapInuse/bytesPerMB,
"heap_released_mb", m.HeapReleased/bytesPerMB,
"stack_inuse_mb", m.StackInuse/bytesPerMB,
)
}
}
// CLIEntry is the main entry point for the CLI
func CLIEntry() {
app := fx.New(
@ -24,6 +61,9 @@ func CLIEntry() {
fx.Invoke(func(lc fx.Lifecycle, rw *RouteWatch, logger *logger.Logger) {
lc.Append(fx.Hook{
OnStart: func(_ context.Context) error {
// Start debug stats logging
go logDebugStats(logger)
go func() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()