Compare commits
	
		
			3 Commits
		
	
	
		
			ddb3cfa4f0
			...
			a78e5c6e92
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
| a78e5c6e92 | |||
| 9ef2a22db3 | |||
| 05805b8847 | 
							
								
								
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
								
							
							
						
						
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
								
							@ -35,3 +35,4 @@ pkg/asinfo/asdata.json
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
# Debug output files
 | 
					# Debug output files
 | 
				
			||||||
out
 | 
					out
 | 
				
			||||||
 | 
					log.txt
 | 
				
			||||||
@ -66,11 +66,9 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 | 
				
			|||||||
	}
 | 
						}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	// Add connection parameters for go-sqlite3
 | 
						// Add connection parameters for go-sqlite3
 | 
				
			||||||
	// Enable WAL mode and other performance optimizations
 | 
					 | 
				
			||||||
	// Configure SQLite connection parameters
 | 
						// Configure SQLite connection parameters
 | 
				
			||||||
	// Use _txlock=immediate to acquire locks upfront and respect busy_timeout
 | 
					 | 
				
			||||||
	dsn := fmt.Sprintf(
 | 
						dsn := fmt.Sprintf(
 | 
				
			||||||
		"file:%s?_busy_timeout=5000&_journal_mode=WAL&_synchronous=OFF&_txlock=immediate",
 | 
							"file:%s",
 | 
				
			||||||
		dbPath,
 | 
							dbPath,
 | 
				
			||||||
	)
 | 
						)
 | 
				
			||||||
	db, err := sql.Open("sqlite3", dsn)
 | 
						db, err := sql.Open("sqlite3", dsn)
 | 
				
			||||||
@ -83,8 +81,7 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 | 
				
			|||||||
	}
 | 
						}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	// Set connection pool parameters
 | 
						// Set connection pool parameters
 | 
				
			||||||
	// Multiple connections for better concurrency
 | 
						// Multiple connections allow concurrent reads while writes are serialized
 | 
				
			||||||
	// Use multiple connections for read concurrency
 | 
					 | 
				
			||||||
	const maxConns = 10
 | 
						const maxConns = 10
 | 
				
			||||||
	db.SetMaxOpenConns(maxConns)
 | 
						db.SetMaxOpenConns(maxConns)
 | 
				
			||||||
	db.SetMaxIdleConns(maxConns)
 | 
						db.SetMaxIdleConns(maxConns)
 | 
				
			||||||
@ -101,22 +98,15 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
// Initialize creates the database schema if it doesn't exist.
 | 
					// Initialize creates the database schema if it doesn't exist.
 | 
				
			||||||
func (d *Database) Initialize() error {
 | 
					func (d *Database) Initialize() error {
 | 
				
			||||||
	// Set SQLite pragmas for extreme performance - prioritize speed over durability
 | 
						// Set SQLite pragmas for performance
 | 
				
			||||||
	pragmas := []string{
 | 
						pragmas := []string{
 | 
				
			||||||
		"PRAGMA journal_mode=WAL",          // Write-Ahead Logging
 | 
							"PRAGMA journal_mode=WAL",         // Write-Ahead Logging
 | 
				
			||||||
		"PRAGMA synchronous=OFF",           // Don't wait for disk writes
 | 
							"PRAGMA synchronous=OFF",          // Don't wait for disk writes
 | 
				
			||||||
		"PRAGMA cache_size=-8388608",       // 8GB cache (negative = KB)
 | 
							"PRAGMA cache_size=-3145728",      // 3GB cache (upper limit for 2.4GB DB)
 | 
				
			||||||
		"PRAGMA temp_store=MEMORY",         // Use memory for temp tables
 | 
							"PRAGMA temp_store=MEMORY",        // Use memory for temp tables
 | 
				
			||||||
		"PRAGMA mmap_size=10737418240",     // 10GB memory-mapped I/O
 | 
							"PRAGMA wal_checkpoint(TRUNCATE)", // Checkpoint and truncate WAL now
 | 
				
			||||||
		"PRAGMA page_size=8192",            // 8KB pages for better performance
 | 
							"PRAGMA busy_timeout=5000",        // 5 second busy timeout
 | 
				
			||||||
		"PRAGMA wal_autocheckpoint=100000", // Checkpoint every 100k pages (800MB)
 | 
							"PRAGMA analysis_limit=0",         // Disable automatic ANALYZE
 | 
				
			||||||
		"PRAGMA wal_checkpoint(TRUNCATE)",  // Checkpoint and truncate WAL now
 | 
					 | 
				
			||||||
		"PRAGMA busy_timeout=5000",         // 5 second busy timeout
 | 
					 | 
				
			||||||
		"PRAGMA locking_mode=NORMAL",       // Normal locking for multiple connections
 | 
					 | 
				
			||||||
		"PRAGMA read_uncommitted=true",     // Allow dirty reads
 | 
					 | 
				
			||||||
		"PRAGMA analysis_limit=0",          // Disable automatic ANALYZE
 | 
					 | 
				
			||||||
		"PRAGMA threads=4",                 // Use multiple threads for sorting
 | 
					 | 
				
			||||||
		"PRAGMA cache_spill=false",         // Keep cache in memory, don't spill to disk
 | 
					 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	for _, pragma := range pragmas {
 | 
						for _, pragma := range pragmas {
 | 
				
			||||||
 | 
				
			|||||||
		Loading…
	
		Reference in New Issue
	
	Block a user