From 7d814c9d2d6b3dc55351ccb906c28cd1f1d1a5b6 Mon Sep 17 00:00:00 2001
From: sneak <sneak@sneak.berlin>
Date: Mon, 28 Jul 2025 02:40:17 +0200
Subject: [PATCH] Optimize SQLite and PrefixHandler for better performance

- Increase PrefixHandler queue size to 500k and batch size to 25k
- Set SQLite PRAGMA synchronous=OFF for faster writes (trades durability)
- Increase SQLite cache to 1GB and mmap to 512MB
- Increase WAL checkpoint interval to 10000 pages
- Set page size to 8KB for better performance
- Increase busy timeout to 30 seconds
- Keep single connection to avoid SQLite locking issues
---
 internal/database/database.go        | 22 +++++++++++++---------
 internal/routewatch/prefixhandler.go |  4 ++--
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/internal/database/database.go b/internal/database/database.go
index c898715..a143b71 100644
--- a/internal/database/database.go
+++ b/internal/database/database.go
@@ -55,7 +55,8 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 	}
 
 	// Set connection pool parameters
-	db.SetMaxOpenConns(1) // Force serialization since SQLite doesn't handle true concurrency well
+	// Single connection to avoid locking issues with SQLite
+	db.SetMaxOpenConns(1)
 	db.SetMaxIdleConns(1)
 	db.SetConnMaxLifetime(0)
 
@@ -71,15 +72,18 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 // Initialize creates the database schema if it doesn't exist.
 func (d *Database) Initialize() error {
 	// Set SQLite pragmas for better performance
+	// WARNING: These settings trade durability for speed
 	pragmas := []string{
-		"PRAGMA journal_mode=WAL",        // Already set in connection string
-		"PRAGMA synchronous=NORMAL",      // Faster than FULL, still safe
-		"PRAGMA cache_size=-524288",      // 512MB cache (negative = KB)
-		"PRAGMA temp_store=MEMORY",       // Use memory for temp tables
-		"PRAGMA mmap_size=268435456",     // 256MB memory-mapped I/O
-		"PRAGMA wal_autocheckpoint=1000", // Checkpoint every 1000 pages
-		"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
-		"PRAGMA optimize",                // Run optimizer
+		"PRAGMA journal_mode=WAL",         // Write-Ahead Logging
+		"PRAGMA synchronous=OFF",          // Don't wait for disk writes - RISKY but FAST
+		"PRAGMA cache_size=-1048576",      // 1GB cache (negative = KB)
+		"PRAGMA temp_store=MEMORY",        // Use memory for temp tables
+		"PRAGMA mmap_size=536870912",      // 512MB memory-mapped I/O
+		"PRAGMA wal_autocheckpoint=10000", // Checkpoint every 10000 pages (less frequent)
+		"PRAGMA wal_checkpoint(PASSIVE)",  // Checkpoint now
+		"PRAGMA page_size=8192",           // Larger page size for better performance
+		"PRAGMA busy_timeout=30000",       // 30 second busy timeout
+		"PRAGMA optimize",                 // Run optimizer
 	}
 
 	for _, pragma := range pragmas {
diff --git a/internal/routewatch/prefixhandler.go b/internal/routewatch/prefixhandler.go
index 1517602..be621e4 100644
--- a/internal/routewatch/prefixhandler.go
+++ b/internal/routewatch/prefixhandler.go
@@ -14,10 +14,10 @@ import (
 
 const (
 	// prefixHandlerQueueSize is the queue capacity for prefix tracking operations
-	prefixHandlerQueueSize = 200000
+	prefixHandlerQueueSize = 500000
 
 	// prefixBatchSize is the number of prefix updates to batch together
-	prefixBatchSize = 10000
+	prefixBatchSize = 25000
 
 	// prefixBatchTimeout is the maximum time to wait before flushing a batch
 	prefixBatchTimeout = 2 * time.Second