Optimize SQLite and PrefixHandler for better performance
- Increase PrefixHandler queue size to 500k and batch size to 25k - Set SQLite PRAGMA synchronous=OFF for faster writes (trades durability) - Increase SQLite cache to 1GB and mmap to 512MB - Increase WAL checkpoint interval to 10000 pages - Set page size to 8KB for better performance - Increase busy timeout to 30 seconds - Keep single connection to avoid SQLite locking issues
This commit is contained in:
parent
54bb0ba1cb
commit
7d814c9d2d
@ -55,7 +55,8 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
|
||||
}
|
||||
|
||||
// Set connection pool parameters
|
||||
db.SetMaxOpenConns(1) // Force serialization since SQLite doesn't handle true concurrency well
|
||||
// Single connection to avoid locking issues with SQLite
|
||||
db.SetMaxOpenConns(1)
|
||||
db.SetMaxIdleConns(1)
|
||||
db.SetConnMaxLifetime(0)
|
||||
|
||||
@ -71,15 +72,18 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
|
||||
// Initialize creates the database schema if it doesn't exist.
|
||||
func (d *Database) Initialize() error {
|
||||
// Set SQLite pragmas for better performance
|
||||
// WARNING: These settings trade durability for speed
|
||||
pragmas := []string{
|
||||
"PRAGMA journal_mode=WAL", // Already set in connection string
|
||||
"PRAGMA synchronous=NORMAL", // Faster than FULL, still safe
|
||||
"PRAGMA cache_size=-524288", // 512MB cache (negative = KB)
|
||||
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
|
||||
"PRAGMA mmap_size=268435456", // 256MB memory-mapped I/O
|
||||
"PRAGMA wal_autocheckpoint=1000", // Checkpoint every 1000 pages
|
||||
"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
|
||||
"PRAGMA optimize", // Run optimizer
|
||||
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
|
||||
"PRAGMA synchronous=OFF", // Don't wait for disk writes - RISKY but FAST
|
||||
"PRAGMA cache_size=-1048576", // 1GB cache (negative = KB)
|
||||
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
|
||||
"PRAGMA mmap_size=536870912", // 512MB memory-mapped I/O
|
||||
"PRAGMA wal_autocheckpoint=10000", // Checkpoint every 10000 pages (less frequent)
|
||||
"PRAGMA wal_checkpoint(PASSIVE)", // Checkpoint now
|
||||
"PRAGMA page_size=8192", // Larger page size for better performance
|
||||
"PRAGMA busy_timeout=30000", // 30 second busy timeout
|
||||
"PRAGMA optimize", // Run optimizer
|
||||
}
|
||||
|
||||
for _, pragma := range pragmas {
|
||||
|
@ -14,10 +14,10 @@ import (
|
||||
|
||||
const (
|
||||
// prefixHandlerQueueSize is the queue capacity for prefix tracking operations
|
||||
prefixHandlerQueueSize = 200000
|
||||
prefixHandlerQueueSize = 500000
|
||||
|
||||
// prefixBatchSize is the number of prefix updates to batch together
|
||||
prefixBatchSize = 10000
|
||||
prefixBatchSize = 25000
|
||||
|
||||
// prefixBatchTimeout is the maximum time to wait before flushing a batch
|
||||
prefixBatchTimeout = 2 * time.Second
|
||||
|
Loading…
Reference in New Issue
Block a user