Optimize SQLite and PrefixHandler for better performance

- Increase PrefixHandler queue size to 500k and batch size to 25k
- Set SQLite PRAGMA synchronous=OFF for faster writes (trades durability)
- Increase SQLite cache to 1GB and mmap to 512MB
- Increase WAL checkpoint interval to 10000 pages
- Set page size to 8KB for better performance
- Increase busy timeout to 30 seconds
- Keep single connection to avoid SQLite locking issues
This commit is contained in:
Jeffrey Paul 2025-07-28 02:40:17 +02:00
parent 54bb0ba1cb
commit 7d814c9d2d
2 changed files with 15 additions and 11 deletions

View File

@ -55,7 +55,8 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
}
// Set connection pool parameters
db.SetMaxOpenConns(1) // Force serialization since SQLite doesn't handle true concurrency well
// Single connection to avoid locking issues with SQLite
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(1)
db.SetConnMaxLifetime(0)
@ -71,15 +72,18 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
// Initialize creates the database schema if it doesn't exist.
func (d *Database) Initialize() error {
// Set SQLite pragmas for better performance
// WARNING: These settings trade durability for speed
pragmas := []string{
"PRAGMA journal_mode=WAL", // Already set in connection string
"PRAGMA synchronous=NORMAL", // Faster than FULL, still safe
"PRAGMA cache_size=-524288", // 512MB cache (negative = KB)
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
"PRAGMA mmap_size=268435456", // 256MB memory-mapped I/O
"PRAGMA wal_autocheckpoint=1000", // Checkpoint every 1000 pages
"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
"PRAGMA optimize", // Run optimizer
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
"PRAGMA synchronous=OFF", // Don't wait for disk writes - RISKY but FAST
"PRAGMA cache_size=-1048576", // 1GB cache (negative = KB)
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
"PRAGMA mmap_size=536870912", // 512MB memory-mapped I/O
"PRAGMA wal_autocheckpoint=10000", // Checkpoint every 10000 pages (less frequent)
"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
"PRAGMA page_size=8192", // Larger page size for better performance
"PRAGMA busy_timeout=30000", // 30 second busy timeout
"PRAGMA optimize", // Run optimizer
}
for _, pragma := range pragmas {

View File

@ -14,10 +14,10 @@ import (
const (
// prefixHandlerQueueSize is the queue capacity for prefix tracking operations
prefixHandlerQueueSize = 200000
prefixHandlerQueueSize = 500000
// prefixBatchSize is the number of prefix updates to batch together
prefixBatchSize = 10000
prefixBatchSize = 25000
// prefixBatchTimeout is the maximum time to wait before flushing a batch
prefixBatchTimeout = 2 * time.Second