Bound SQLite memory across the whole connection pool (closes #8)

The 3 GiB page cache and temp_store=MEMORY were set once in Initialize, so
only one pooled connection carried them and the other nine got no busy_timeout,
which drove many "database is locked" errors.

Move the per-connection settings into the DSN so every pooled connection gets a
64 MiB cache (640 MiB worst case over ten connections), synchronous OFF, a
5 s busy_timeout and WAL. Drop those pragmas from Initialize; DISTINCT temp
B-trees now spill to disk. Add process-wide soft (1 GiB) and hard (1.5 GiB)
heap limits; at the hard limit a statement returns SQLITE_NOMEM and the
existing batch paths log and drop, so nothing panics or exits.

Model: opus-4-8
This commit was merged in pull request #15.
This commit is contained in:
2026-09-21 15:46:34 +02:00
parent 54014c88c8
commit 594e7a504b
2 changed files with 107 additions and 8 deletions
+28 -8
View File
@@ -38,6 +38,22 @@ const (
maxIPv4 = 0xFFFFFFFF
)
// SQLite memory tuning. cache_size and busy_timeout go in the DSN so every
// pooled connection gets them; the heap limits are process-wide and set once.
const (
// sqliteCacheSizeKiB is the per-connection page cache; negative means KiB.
// -65536 = 64 MiB, so at most 640 MiB across the 10-connection pool.
sqliteCacheSizeKiB = -65536
// sqliteBusyTimeoutMs is how long a connection waits on a locked database.
sqliteBusyTimeoutMs = 5000
// sqliteSoftHeapLimitBytes (1 GiB) makes SQLite recycle its cache rather
// than allocate once its C heap passes this size.
sqliteSoftHeapLimitBytes = 1073741824
// sqliteHardHeapLimitBytes (1.5 GiB) fails a statement with SQLITE_NOMEM
// instead of growing the C heap without bound.
sqliteHardHeapLimitBytes = 1610612736
)
// Common errors
var (
// ErrInvalidIP is returned when an IP address is malformed
@@ -71,11 +87,13 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
// Add connection parameters for go-sqlite3
// Configure SQLite connection parameters
// Per-connection SQLite settings go in the DSN so every pooled connection
// gets them, not just the one that runs the Initialize pragmas.
dsn := fmt.Sprintf(
"file:%s",
"file:%s?_cache_size=%d&_synchronous=OFF&_busy_timeout=%d&_journal_mode=WAL",
dbPath,
sqliteCacheSizeKiB,
sqliteBusyTimeoutMs,
)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
@@ -104,15 +122,17 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
// Initialize creates the database schema if it doesn't exist.
func (d *Database) Initialize() error {
// Set SQLite pragmas for performance
// Set SQLite pragmas for performance. Per-connection settings (cache_size,
// synchronous, busy_timeout, journal_mode) live in the DSN; temp_store is
// left at its default so DISTINCT temp B-trees spill to disk instead of C
// heap. The heap limits below are process-wide, so setting them once here is
// enough for the whole pool.
pragmas := []string{
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
"PRAGMA synchronous=OFF", // Don't wait for disk writes
"PRAGMA cache_size=-3145728", // 3GB cache (upper limit for 2.4GB DB)
"PRAGMA temp_store=MEMORY", // Use memory for temp tables
"PRAGMA busy_timeout=5000", // 5 second busy timeout
"PRAGMA analysis_limit=0", // Disable automatic ANALYZE
"PRAGMA auto_vacuum=INCREMENTAL", // Enable incremental vacuum
fmt.Sprintf("PRAGMA soft_heap_limit=%d", sqliteSoftHeapLimitBytes),
fmt.Sprintf("PRAGMA hard_heap_limit=%d", sqliteHardHeapLimitBytes),
}
for _, pragma := range pragmas {