.gitignore

@@ -35,4 +35,3 @@ pkg/asinfo/asdata.json
 
 # Debug output files
 out
-log.txt

internal/database/database.go

@@ -66,9 +66,11 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 	}
 
 	// Add connection parameters for go-sqlite3
+	// Enable WAL mode and other performance optimizations
 	// Configure SQLite connection parameters
+	// Use _txlock=immediate to acquire locks upfront and respect busy_timeout
 	dsn := fmt.Sprintf(
-		"file:%s",
+		"file:%s?_busy_timeout=5000&_journal_mode=WAL&_synchronous=OFF&_txlock=immediate",
 		dbPath,
 	)
 	db, err := sql.Open("sqlite3", dsn)
@@ -81,7 +83,8 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 	}
 
 	// Set connection pool parameters
-	// Multiple connections allow concurrent reads while writes are serialized
+	// Multiple connections for better concurrency
+	// Use multiple connections for read concurrency
 	const maxConns = 10
 	db.SetMaxOpenConns(maxConns)
 	db.SetMaxIdleConns(maxConns)
@@ -98,15 +101,22 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 
 // Initialize creates the database schema if it doesn't exist.
 func (d *Database) Initialize() error {
-	// Set SQLite pragmas for performance
+	// Set SQLite pragmas for extreme performance - prioritize speed over durability
 	pragmas := []string{
-		"PRAGMA journal_mode=WAL",         // Write-Ahead Logging
+		"PRAGMA journal_mode=WAL",          // Write-Ahead Logging
-		"PRAGMA synchronous=OFF",          // Don't wait for disk writes
+		"PRAGMA synchronous=OFF",           // Don't wait for disk writes
-		"PRAGMA cache_size=-3145728",      // 3GB cache (upper limit for 2.4GB DB)
+		"PRAGMA cache_size=-8388608",       // 8GB cache (negative = KB)
-		"PRAGMA temp_store=MEMORY",        // Use memory for temp tables
+		"PRAGMA temp_store=MEMORY",         // Use memory for temp tables
-		"PRAGMA wal_checkpoint(TRUNCATE)", // Checkpoint and truncate WAL now
+		"PRAGMA mmap_size=10737418240",     // 10GB memory-mapped I/O
-		"PRAGMA busy_timeout=5000",        // 5 second busy timeout
+		"PRAGMA page_size=8192",            // 8KB pages for better performance
-		"PRAGMA analysis_limit=0",         // Disable automatic ANALYZE
+		"PRAGMA wal_autocheckpoint=100000", // Checkpoint every 100k pages (800MB)
+		"PRAGMA wal_checkpoint(TRUNCATE)",  // Checkpoint and truncate WAL now
+		"PRAGMA busy_timeout=5000",         // 5 second busy timeout
+		"PRAGMA locking_mode=NORMAL",       // Normal locking for multiple connections
+		"PRAGMA read_uncommitted=true",     // Allow dirty reads
+		"PRAGMA analysis_limit=0",          // Disable automatic ANALYZE
+		"PRAGMA threads=4",                 // Use multiple threads for sorting
+		"PRAGMA cache_spill=false",         // Keep cache in memory, don't spill to disk
 	}
 
 	for _, pragma := range pragmas {