diff --git a/internal/database/database.go b/internal/database/database.go index c898715..a143b71 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -55,7 +55,8 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { } // Set connection pool parameters - db.SetMaxOpenConns(1) // Force serialization since SQLite doesn't handle true concurrency well + // Single connection to avoid locking issues with SQLite + db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) db.SetConnMaxLifetime(0) @@ -71,15 +72,18 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { // Initialize creates the database schema if it doesn't exist. func (d *Database) Initialize() error { // Set SQLite pragmas for better performance + // WARNING: These settings trade durability for speed pragmas := []string{ - "PRAGMA journal_mode=WAL", // Already set in connection string - "PRAGMA synchronous=NORMAL", // Faster than FULL, still safe - "PRAGMA cache_size=-524288", // 512MB cache (negative = KB) - "PRAGMA temp_store=MEMORY", // Use memory for temp tables - "PRAGMA mmap_size=268435456", // 256MB memory-mapped I/O - "PRAGMA wal_autocheckpoint=1000", // Checkpoint every 1000 pages - "PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now - "PRAGMA optimize", // Run optimizer + "PRAGMA journal_mode=WAL", // Write-Ahead Logging + "PRAGMA synchronous=OFF", // Don't wait for disk writes - RISKY but FAST + "PRAGMA cache_size=-1048576", // 1GB cache (negative = KB) + "PRAGMA temp_store=MEMORY", // Use memory for temp tables + "PRAGMA mmap_size=536870912", // 512MB memory-mapped I/O + "PRAGMA wal_autocheckpoint=10000", // Checkpoint every 10000 pages (less frequent) + "PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now + "PRAGMA page_size=8192", // Larger page size for better performance + "PRAGMA busy_timeout=30000", // 30 second busy timeout + "PRAGMA optimize", // Run optimizer } for _, pragma := range pragmas { diff --git a/internal/routewatch/prefixhandler.go b/internal/routewatch/prefixhandler.go index 1517602..be621e4 100644 --- a/internal/routewatch/prefixhandler.go +++ b/internal/routewatch/prefixhandler.go @@ -14,10 +14,10 @@ import ( const ( // prefixHandlerQueueSize is the queue capacity for prefix tracking operations - prefixHandlerQueueSize = 200000 + prefixHandlerQueueSize = 500000 // prefixBatchSize is the number of prefix updates to batch together - prefixBatchSize = 10000 + prefixBatchSize = 25000 // prefixBatchTimeout is the maximum time to wait before flushing a batch prefixBatchTimeout = 2 * time.Second