Compare commits

...

3 Commits

Author SHA1 Message Date
a78e5c6e92 Add log.txt to .gitignore 2025-07-28 18:13:07 +02:00
9ef2a22db3 Remove SQLite pragmas that set default values
- Remove page_size, wal_autocheckpoint, locking_mode, mmap_size
- Keep only pragmas that change behavior from defaults
- Increase cache size to 3GB (upper limit for 2.4GB database)
2025-07-28 18:12:25 +02:00
05805b8847 Optimize SQLite settings for better balance
- Reduce cache size from 8GB to 512MB (still plenty for 2.4GB DB)
- Reduce mmap_size from 10GB to 256MB (reasonable default)
- Use default page size (4KB) instead of 8KB
- Use default WAL checkpoint interval (1000 pages)
- Remove redundant pragmas (threads, cache_spill, read_uncommitted)
- Clean up connection string to only use _txlock parameter
- Keep synchronous=OFF for performance (since we have mutex protection)
2025-07-28 18:06:31 +02:00
3 changed files with 270945 additions and 119 deletions

3
.gitignore vendored
View File

@ -34,4 +34,5 @@ go.work.sum
pkg/asinfo/asdata.json
# Debug output files
out
out
log.txt

View File

@ -66,11 +66,9 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
}
// Add connection parameters for go-sqlite3
// Enable WAL mode and other performance optimizations
// Configure SQLite connection parameters
// Use _txlock=immediate to acquire locks upfront and respect busy_timeout
dsn := fmt.Sprintf(
"file:%s?_busy_timeout=5000&_journal_mode=WAL&_synchronous=OFF&_txlock=immediate",
"file:%s",
dbPath,
)
db, err := sql.Open("sqlite3", dsn)
@ -83,8 +81,7 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
}
// Set connection pool parameters
// Multiple connections for better concurrency
// Use multiple connections for read concurrency
// Multiple connections allow concurrent reads while writes are serialized
const maxConns = 10
db.SetMaxOpenConns(maxConns)
db.SetMaxIdleConns(maxConns)
@ -101,22 +98,15 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
// Initialize creates the database schema if it doesn't exist.
func (d *Database) Initialize() error {
// Set SQLite pragmas for extreme performance - prioritize speed over durability
// Set SQLite pragmas for performance
pragmas := []string{
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
"PRAGMA synchronous=OFF", // Don't wait for disk writes
"PRAGMA cache_size=-8388608", // 8GB cache (negative = KB)
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
"PRAGMA mmap_size=10737418240", // 10GB memory-mapped I/O
"PRAGMA page_size=8192", // 8KB pages for better performance
"PRAGMA wal_autocheckpoint=100000", // Checkpoint every 100k pages (800MB)
"PRAGMA wal_checkpoint(TRUNCATE)", // Checkpoint and truncate WAL now
"PRAGMA busy_timeout=5000", // 5 second busy timeout
"PRAGMA locking_mode=NORMAL", // Normal locking for multiple connections
"PRAGMA read_uncommitted=true", // Allow dirty reads
"PRAGMA analysis_limit=0", // Disable automatic ANALYZE
"PRAGMA threads=4", // Use multiple threads for sorting
"PRAGMA cache_spill=false", // Keep cache in memory, don't spill to disk
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
"PRAGMA synchronous=OFF", // Don't wait for disk writes
"PRAGMA cache_size=-3145728", // 3GB cache (upper limit for 2.4GB DB)
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
"PRAGMA wal_checkpoint(TRUNCATE)", // Checkpoint and truncate WAL now
"PRAGMA busy_timeout=5000", // 5 second busy timeout
"PRAGMA analysis_limit=0", // Disable automatic ANALYZE
}
for _, pragma := range pragmas {

271031
log.txt

File diff suppressed because it is too large Load Diff