Optimize SQLite and PrefixHandler for better performance

- Increase PrefixHandler queue size to 500k and batch size to 25k
- Set SQLite PRAGMA synchronous=OFF for faster writes (trades durability)
- Increase SQLite cache to 1GB and mmap to 512MB
- Increase WAL checkpoint interval to 10000 pages
- Set page size to 8KB for better performance
- Increase busy timeout to 30 seconds
- Keep single connection to avoid SQLite locking issues

internal/database/database.go

@@ -55,7 +55,8 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 	}
 
 	// Set connection pool parameters
-	db.SetMaxOpenConns(1) // Force serialization since SQLite doesn't handle true concurrency well
+	// Single connection to avoid locking issues with SQLite
+	db.SetMaxOpenConns(1)
 	db.SetMaxIdleConns(1)
 	db.SetConnMaxLifetime(0)
 
@@ -71,15 +72,18 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 // Initialize creates the database schema if it doesn't exist.
 func (d *Database) Initialize() error {
 	// Set SQLite pragmas for better performance
+	// WARNING: These settings trade durability for speed
 	pragmas := []string{
-		"PRAGMA journal_mode=WAL",        // Already set in connection string
+		"PRAGMA journal_mode=WAL",         // Write-Ahead Logging
-		"PRAGMA synchronous=NORMAL",      // Faster than FULL, still safe
+		"PRAGMA synchronous=OFF",          // Don't wait for disk writes - RISKY but FAST
-		"PRAGMA cache_size=-524288",      // 512MB cache (negative = KB)
+		"PRAGMA cache_size=-1048576",      // 1GB cache (negative = KB)
-		"PRAGMA temp_store=MEMORY",       // Use memory for temp tables
+		"PRAGMA temp_store=MEMORY",        // Use memory for temp tables
-		"PRAGMA mmap_size=268435456",     // 256MB memory-mapped I/O
+		"PRAGMA mmap_size=536870912",      // 512MB memory-mapped I/O
-		"PRAGMA wal_autocheckpoint=1000", // Checkpoint every 1000 pages
+		"PRAGMA wal_autocheckpoint=10000", // Checkpoint every 10000 pages (less frequent)
-		"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
+		"PRAGMA wal_checkpoint(PASSIVE)",  // Checkpoint now
-		"PRAGMA optimize",                // Run optimizer
+		"PRAGMA page_size=8192",           // Larger page size for better performance
+		"PRAGMA busy_timeout=30000",       // 30 second busy timeout
+		"PRAGMA optimize",                 // Run optimizer
 	}
 
 	for _, pragma := range pragmas {

internal/routewatch/prefixhandler.go

@@ -14,10 +14,10 @@ import (
 
 const (
 	// prefixHandlerQueueSize is the queue capacity for prefix tracking operations
-	prefixHandlerQueueSize = 200000
+	prefixHandlerQueueSize = 500000
 
 	// prefixBatchSize is the number of prefix updates to batch together
-	prefixBatchSize = 10000
+	prefixBatchSize = 25000
 
 	// prefixBatchTimeout is the maximum time to wait before flushing a batch
 	prefixBatchTimeout = 2 * time.Second