package database import ( "database/sql" "errors" "fmt" "io/fs" "net/url" "os" "time" _ "modernc.org/sqlite" // Pure Go SQLite driver ) // Every SQLite file this service opens — the main database, the // per-webhook event databases, and the archive databases — is opened // through OpenSQLite, so the durability settings below are properties // of the service rather than of one call site. // // modernc.org/sqlite installs no busy handler and issues no pragmas of // its own: it executes only the pragmas named in explicit `_pragma=` // DSN parameters, and gorm.io/driver/sqlite adds none when it is // handed an existing *sql.DB. Every setting therefore has to be // spelled out here or it is simply not in effect. // SQLite URI open modes. const ( // SQLiteModeCreate creates the database file when it is missing. SQLiteModeCreate = "rwc" // SQLiteModeExisting requires the file to exist already. SQLiteModeExisting = "rw" ) const ( // SQLiteBusyTimeout is how long SQLite retries a lock conflict // before returning SQLITE_BUSY. // // Under WAL a reader never blocks a writer, so the only conflict // left is writer against writer: this process's delivery workers // against each other, or against another process holding the write // lock. Those clear in milliseconds. Ten seconds is far above that // and still well inside the receiver's request budget, so an // inbound webhook waits rather than being rejected with a 500. SQLiteBusyTimeout = 10 * time.Second // sqliteMaxOpenConns bounds the connection pool for one database // file. // // The pool needs a bound at all because database/sql cannot detect // a connection left mid-transaction: modernc.org/sqlite implements // neither driver.Validator nor driver.SessionResetter, so a // connection whose COMMIT failed is returned to the pool with its // transaction still open and handed out again indefinitely. That is // what turned four `database is locked` errors into 593 // `cannot start a transaction within a transaction` in // https://git.eeqj.de/sneak/webhooker/issues/256. // // Four is above the one writer SQLite allows at a time, so reads // still proceed while a write is in flight, and low enough that // contention is resolved by the busy handler rather than by piling // up connections against a lock only one of them can hold. sqliteMaxOpenConns = 4 // sqliteMaxIdleConns keeps the pool warm without holding every // connection open through an idle period. sqliteMaxIdleConns = 2 // sqliteConnMaxLifetime and sqliteConnMaxIdleTime retire pooled // connections on a schedule. With _txlock=immediate a failed // COMMIT should no longer be reachable, but these bound the damage // if one happens anyway: a poisoned connection is closed and // replaced within the lifetime instead of wedging the file until // the process restarts. sqliteConnMaxLifetime = 5 * time.Minute sqliteConnMaxIdleTime = time.Minute ) // SQLiteFilePerm is the mode every SQLite file this service owns is // created with and held at: owner read/write, nothing for group or // other. // // These files hold credentials in plaintext. The main database stores // `targets.config` — bearer tokens, API keys, Slack webhook URLs — and // the session encryption key. SQLite left to itself creates them 0644 // (see reserveSQLiteFile), which made the 0750 data directory the only // barrier; a bind-mounted directory supplied at 0755 removes it and // every local user on the host can read every stored credential. // // This is a file-mode fix and not encryption at rest. An unattended // process needs a key it can read without a human, so the key lands // beside the data and an attacker who can read the database can read // it too. See https://git.eeqj.de/sneak/webhooker/issues/212. const SQLiteFilePerm fs.FileMode = 0o600 // reserveSQLiteFile puts path at SQLiteFilePerm before the driver ever // touches it, and tightens any sidecar already on disk. // // The mode has to be settled here rather than by a chmod after opening, // because SQLite picks it: robust_open substitutes // SQLITE_DEFAULT_FILE_PERMISSIONS (0644) whenever it is handed mode 0, // and findCreateFileMode yields 0 for a main database opened by URI // with no `modeof` parameter. A chmod afterwards would leave a window // in which the credentials are on disk world-readable. // // Creating the file ourselves also settles the sidecars, which is the // half that could quietly not work. SQLite does not create those at a // mode we choose — it derives both from the main database file: // `-wal` through findCreateFileMode, which stats the path with the // suffix stripped, and `-shm` in unixOpenSharedMemory from an fstat of // the already-open database descriptor. A main file at 0600 therefore // produces sidecars at 0600. A zero-length file is a valid empty // database, so reserving it changes nothing else. // // create says whether the caller is opening in a mode that may create // the database. When it is false a missing file is left missing, so // SQLite still reports the absence rather than this function // materializing an empty database the caller asked not to create. // // Chmod of a file that already exists is what tightens a data // directory an earlier build left at 0644 — including a developer's // own scratch directory — without any migration machinery. func reserveSQLiteFile(path string, create bool) error { if create { // gosec G304: the path is the database file the caller asked // to open, and the driver is about to open the same path // anyway. Creating it here is what fixes its mode. f, err := os.OpenFile( //nolint:gosec // see above path, os.O_RDWR|os.O_CREATE, SQLiteFilePerm, ) if err != nil { return fmt.Errorf("creating %s: %w", path, err) } err = f.Close() if err != nil { return fmt.Errorf("closing %s: %w", path, err) } } // O_CREATE leaves an existing file's mode alone, and umask can only // have narrowed a new one. Chmod settles both cases at exactly // SQLiteFilePerm. for _, p := range append( []string{path}, sqliteSidecarPaths(path)..., ) { err := os.Chmod(p, SQLiteFilePerm) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("securing %s: %w", p, err) } } return nil } // sqliteSidecarPaths returns the files SQLite maintains beside a // database under WAL. They carry the same rows as the database itself, // so a fix that tightens only the main file has fixed nothing. func sqliteSidecarPaths(path string) []string { return []string{path + "-wal", path + "-shm"} } // SQLiteDSN builds the connection string for one database file. // // mode is the SQLite URI open mode: "rwc" to create the file when it // is missing, "rw" to require that it already exists. // // Three settings carry the fix for // https://git.eeqj.de/sneak/webhooker/issues/256 and none of them is // optional: // // - journal_mode=WAL, so a reader — an operator running // `sqlite3 .dump` over their own data — takes a snapshot // instead of blocking every writer behind it. // // - busy_timeout, so a writer that does meet a lock waits for it. // Without one SQLite gives up immediately; nothing above it // retries. // // - _txlock=immediate, so every transaction takes the write lock at // BEGIN. A deferred transaction acquires it lazily on its first // write, and that upgrade returns SQLITE_BUSY *without* consulting // the busy handler, because SQLite cannot block a transaction that // may already hold a read snapshot. Such a COMMIT then fails while // the transaction stays open on the connection. A busy timeout // alone does not prevent this; BEGIN IMMEDIATE does, by putting // the wait somewhere the handler applies. // // Note what is absent: `cache=shared`. Under a shared cache an // in-process conflict is reported as SQLITE_LOCKED rather than // SQLITE_BUSY, and the busy handler does not retry SQLITE_LOCKED — so // leaving it in would have defeated the busy timeout for exactly the // contention this service generates. Dropping it is part of the fix, // not housekeeping. // // synchronous is deliberately left at SQLite's default of FULL: this // is a webhook receiver whose one promise is that an event it answered // 200 for is durable. // The order of the _pragma parameters is load-bearing. // modernc.org/sqlite executes them in the order they appear, on every // new connection, before the connection is handed to the pool. Setting // journal_mode first means that pragma itself runs with no busy // handler installed: the pool opens connections lazily, so the moment // a new one is created is a moment the database is under load, and // PRAGMA journal_mode takes a lock. It would fail immediately with // SQLITE_BUSY and fail the query that caused the connection to be // opened. busy_timeout is therefore set first, so every pragma after // it — and the whole life of the connection — is covered. func SQLiteDSN(path, mode string) string { q := url.Values{} q.Set("mode", mode) q.Set("_txlock", "immediate") q.Add( "_pragma", fmt.Sprintf( "busy_timeout(%d)", SQLiteBusyTimeout.Milliseconds(), ), ) q.Add("_pragma", "journal_mode(WAL)") return "file:" + path + "?" + q.Encode() } // OpenSQLite opens the SQLite file at path with the service's // durability settings and pool bounds applied. mode is the SQLite URI // open mode ("rwc" or "rw"). // // The file and its WAL sidecars are settled at SQLiteFilePerm before // the driver sees the path; see reserveSQLiteFile. // // The handle is returned rather than a *gorm.DB because the callers // wrap it in gorm themselves with their own logger. func OpenSQLite(path, mode string) (*sql.DB, error) { err := reserveSQLiteFile(path, mode == SQLiteModeCreate) if err != nil { return nil, err } sqlDB, err := sql.Open("sqlite", SQLiteDSN(path, mode)) if err != nil { return nil, fmt.Errorf( "opening sqlite database %s: %w", path, err, ) } sqlDB.SetMaxOpenConns(sqliteMaxOpenConns) sqlDB.SetMaxIdleConns(sqliteMaxIdleConns) sqlDB.SetConnMaxLifetime(sqliteConnMaxLifetime) sqlDB.SetConnMaxIdleTime(sqliteConnMaxIdleTime) return sqlDB, nil }