Add debug logging and optimize SQLite performance
- Add debug logging for goroutines and memory usage (enabled via DEBUG=routewatch) - Increase SQLite connection pool from 1 to 10 connections for better concurrency - Optimize SQLite pragmas for balanced performance and safety - Add proper shutdown handling for peering handler - Define constants to avoid magic numbers in code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
9b649c98c9
commit
78d6e17c76
@ -30,6 +30,11 @@ const (
|
||||
ipv4Offset = 12
|
||||
ipv4Bits = 32
|
||||
maxIPv4 = 0xFFFFFFFF
|
||||
|
||||
// Database connection pool settings
|
||||
maxOpenConns = 10
|
||||
maxIdleConns = 5
|
||||
connMaxLifetime = 5 * time.Minute
|
||||
)
|
||||
|
||||
// Common errors
|
||||
@ -73,10 +78,10 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
|
||||
}
|
||||
|
||||
// Set connection pool parameters
|
||||
// Single connection to avoid locking issues with SQLite
|
||||
db.SetMaxOpenConns(1)
|
||||
db.SetMaxIdleConns(1)
|
||||
db.SetConnMaxLifetime(0)
|
||||
// Allow multiple readers but single writer for SQLite WAL mode
|
||||
db.SetMaxOpenConns(maxOpenConns)
|
||||
db.SetMaxIdleConns(maxIdleConns)
|
||||
db.SetConnMaxLifetime(connMaxLifetime)
|
||||
|
||||
database := &Database{db: db, logger: logger, path: dbPath}
|
||||
|
||||
@ -89,19 +94,20 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
|
||||
|
||||
// Initialize creates the database schema if it doesn't exist.
|
||||
func (d *Database) Initialize() error {
|
||||
// Set SQLite pragmas for better performance
|
||||
// WARNING: These settings trade durability for speed
|
||||
// Set SQLite pragmas for better performance with safety
|
||||
pragmas := []string{
|
||||
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
|
||||
"PRAGMA synchronous=OFF", // Don't wait for disk writes - RISKY but FAST
|
||||
"PRAGMA cache_size=-1048576", // 1GB cache (negative = KB)
|
||||
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
|
||||
"PRAGMA mmap_size=536870912", // 512MB memory-mapped I/O
|
||||
"PRAGMA wal_autocheckpoint=10000", // Checkpoint every 10000 pages (less frequent)
|
||||
"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
|
||||
"PRAGMA page_size=8192", // Larger page size for better performance
|
||||
"PRAGMA busy_timeout=30000", // 30 second busy timeout
|
||||
"PRAGMA optimize", // Run optimizer
|
||||
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
|
||||
"PRAGMA synchronous=NORMAL", // Balance between safety and speed
|
||||
"PRAGMA cache_size=-262144", // 256MB cache (negative = KB)
|
||||
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
|
||||
"PRAGMA mmap_size=268435456", // 256MB memory-mapped I/O
|
||||
"PRAGMA wal_autocheckpoint=1000", // Checkpoint every 1000 pages
|
||||
"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
|
||||
"PRAGMA page_size=4096", // Standard page size
|
||||
"PRAGMA busy_timeout=30000", // 30 second busy timeout
|
||||
"PRAGMA optimize", // Run optimizer
|
||||
"PRAGMA locking_mode=NORMAL", // Allow multiple connections
|
||||
"PRAGMA read_uncommitted=true", // Allow dirty reads for better concurrency
|
||||
}
|
||||
|
||||
for _, pragma := range pragmas {
|
||||
|
@ -139,6 +139,10 @@ func (rw *RouteWatch) Shutdown() {
|
||||
rw.logger.Info("Flushing prefix handler")
|
||||
rw.prefixHandler.Stop()
|
||||
}
|
||||
if rw.peeringHandler != nil {
|
||||
rw.logger.Info("Flushing peering handler")
|
||||
rw.peeringHandler.Stop()
|
||||
}
|
||||
|
||||
// Stop services
|
||||
rw.streamer.Stop()
|
||||
|
@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@ -14,8 +16,43 @@ import (
|
||||
const (
|
||||
// shutdownTimeout is the maximum time allowed for graceful shutdown
|
||||
shutdownTimeout = 60 * time.Second
|
||||
// debugInterval is how often to log debug stats
|
||||
debugInterval = 60 * time.Second
|
||||
// bytesPerMB is bytes per megabyte
|
||||
bytesPerMB = 1024 * 1024
|
||||
)
|
||||
|
||||
// logDebugStats logs goroutine count and memory usage
|
||||
func logDebugStats(logger *logger.Logger) {
|
||||
// Only run if DEBUG env var contains "routewatch"
|
||||
debugEnv := os.Getenv("DEBUG")
|
||||
if !strings.Contains(debugEnv, "routewatch") {
|
||||
return
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(debugInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
|
||||
logger.Debug("System stats",
|
||||
"goroutines", runtime.NumGoroutine(),
|
||||
"alloc_mb", m.Alloc/bytesPerMB,
|
||||
"total_alloc_mb", m.TotalAlloc/bytesPerMB,
|
||||
"sys_mb", m.Sys/bytesPerMB,
|
||||
"num_gc", m.NumGC,
|
||||
"heap_alloc_mb", m.HeapAlloc/bytesPerMB,
|
||||
"heap_sys_mb", m.HeapSys/bytesPerMB,
|
||||
"heap_idle_mb", m.HeapIdle/bytesPerMB,
|
||||
"heap_inuse_mb", m.HeapInuse/bytesPerMB,
|
||||
"heap_released_mb", m.HeapReleased/bytesPerMB,
|
||||
"stack_inuse_mb", m.StackInuse/bytesPerMB,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// CLIEntry is the main entry point for the CLI
|
||||
func CLIEntry() {
|
||||
app := fx.New(
|
||||
@ -24,6 +61,9 @@ func CLIEntry() {
|
||||
fx.Invoke(func(lc fx.Lifecycle, rw *RouteWatch, logger *logger.Logger) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
// Start debug stats logging
|
||||
go logDebugStats(logger)
|
||||
|
||||
go func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
Loading…
Reference in New Issue
Block a user