webhooker/internal/database/sqlite_open.go

package database

import (
	"database/sql"
	"errors"
	"fmt"
	"io/fs"
	"net/url"
	"os"
	"time"

	_ "modernc.org/sqlite" // Pure Go SQLite driver
)

// Every SQLite file this service opens — the main database, the
// per-webhook event databases, and the archive databases — is opened
// through OpenSQLite, so the durability settings below are properties
// of the service rather than of one call site.
//
// modernc.org/sqlite installs no busy handler and issues no pragmas of
// its own: it executes only the pragmas named in explicit `_pragma=`
// DSN parameters, and gorm.io/driver/sqlite adds none when it is
// handed an existing *sql.DB. Every setting therefore has to be
// spelled out here or it is simply not in effect.
// SQLite URI open modes.
const (
	// SQLiteModeCreate creates the database file when it is missing.
	SQLiteModeCreate = "rwc"

	// SQLiteModeExisting requires the file to exist already.
	SQLiteModeExisting = "rw"
)

const (
	// SQLiteBusyTimeout is how long SQLite retries a lock conflict
	// before returning SQLITE_BUSY.
	//
	// Under WAL a reader never blocks a writer, so the only conflict
	// left is writer against writer: this process's delivery workers
	// against each other, or against another process holding the write
	// lock. Those clear in milliseconds. Ten seconds is far above that
	// and still well inside the receiver's request budget, so an
	// inbound webhook waits rather than being rejected with a 500.
	SQLiteBusyTimeout = 10 * time.Second

	// sqliteMaxOpenConns bounds the connection pool for one database
	// file.
	//
	// The pool needs a bound at all because database/sql cannot detect
	// a connection left mid-transaction: modernc.org/sqlite implements
	// neither driver.Validator nor driver.SessionResetter, so a
	// connection whose COMMIT failed is returned to the pool with its
	// transaction still open and handed out again indefinitely. That is
	// what turned four `database is locked` errors into 593
	// `cannot start a transaction within a transaction` in
	// https://git.eeqj.de/sneak/webhooker/issues/256.
	//
	// Four is above the one writer SQLite allows at a time, so reads
	// still proceed while a write is in flight, and low enough that
	// contention is resolved by the busy handler rather than by piling
	// up connections against a lock only one of them can hold.
	sqliteMaxOpenConns = 4

	// sqliteMaxIdleConns keeps the pool warm without holding every
	// connection open through an idle period.
	sqliteMaxIdleConns = 2

	// sqliteConnMaxLifetime and sqliteConnMaxIdleTime retire pooled
	// connections on a schedule. With _txlock=immediate a failed
	// COMMIT should no longer be reachable, but these bound the damage
	// if one happens anyway: a poisoned connection is closed and
	// replaced within the lifetime instead of wedging the file until
	// the process restarts.
	sqliteConnMaxLifetime = 5 * time.Minute
	sqliteConnMaxIdleTime = time.Minute
)

// SQLiteFilePerm is the mode every SQLite file this service owns is
// created with and held at: owner read/write, nothing for group or
// other.
//
// These files hold credentials in plaintext. The main database stores
// `targets.config` — bearer tokens, API keys, Slack webhook URLs — and
// the session encryption key. SQLite left to itself creates them 0644
// (see reserveSQLiteFile), which made the 0750 data directory the only
// barrier; a bind-mounted directory supplied at 0755 removes it and
// every local user on the host can read every stored credential.
//
// This is a file-mode fix and not encryption at rest. An unattended
// process needs a key it can read without a human, so the key lands
// beside the data and an attacker who can read the database can read
// it too. See https://git.eeqj.de/sneak/webhooker/issues/212.
const SQLiteFilePerm fs.FileMode = 0o600

// reserveSQLiteFile puts path at SQLiteFilePerm before the driver ever
// touches it, and tightens any sidecar already on disk.
//
// The mode has to be settled here rather than by a chmod after opening,
// because SQLite picks it: robust_open substitutes
// SQLITE_DEFAULT_FILE_PERMISSIONS (0644) whenever it is handed mode 0,
// and findCreateFileMode yields 0 for a main database opened by URI
// with no `modeof` parameter. A chmod afterwards would leave a window
// in which the credentials are on disk world-readable.
//
// Creating the file ourselves also settles the sidecars, which is the
// half that could quietly not work. SQLite does not create those at a
// mode we choose — it derives both from the main database file:
// `-wal` through findCreateFileMode, which stats the path with the
// suffix stripped, and `-shm` in unixOpenSharedMemory from an fstat of
// the already-open database descriptor. A main file at 0600 therefore
// produces sidecars at 0600. A zero-length file is a valid empty
// database, so reserving it changes nothing else.
//
// create says whether the caller is opening in a mode that may create
// the database. When it is false a missing file is left missing, so
// SQLite still reports the absence rather than this function
// materializing an empty database the caller asked not to create.
//
// Chmod of a file that already exists is what tightens a data
// directory an earlier build left at 0644 — including a developer's
// own scratch directory — without any migration machinery.
func reserveSQLiteFile(path string, create bool) error {
	if create {
		// gosec G304: the path is the database file the caller asked
		// to open, and the driver is about to open the same path
		// anyway. Creating it here is what fixes its mode.
		f, err := os.OpenFile( //nolint:gosec // see above
			path, os.O_RDWR|os.O_CREATE, SQLiteFilePerm,
		)
		if err != nil {
			return fmt.Errorf("creating %s: %w", path, err)
		}

		err = f.Close()
		if err != nil {
			return fmt.Errorf("closing %s: %w", path, err)
		}
	}

	// O_CREATE leaves an existing file's mode alone, and umask can only
	// have narrowed a new one. Chmod settles both cases at exactly
	// SQLiteFilePerm.
	for _, p := range append(
		[]string{path}, sqliteSidecarPaths(path)...,
	) {
		err := os.Chmod(p, SQLiteFilePerm)
		if err != nil && !errors.Is(err, fs.ErrNotExist) {
			return fmt.Errorf("securing %s: %w", p, err)
		}
	}

	return nil
}

// sqliteSidecarPaths returns the files SQLite maintains beside a
// database under WAL. They carry the same rows as the database itself,
// so a fix that tightens only the main file has fixed nothing.
func sqliteSidecarPaths(path string) []string {
	return []string{path + "-wal", path + "-shm"}
}

// SQLiteDSN builds the connection string for one database file.
//
// mode is the SQLite URI open mode: "rwc" to create the file when it
// is missing, "rw" to require that it already exists.
//
// Three settings carry the fix for
// https://git.eeqj.de/sneak/webhooker/issues/256 and none of them is
// optional:
//
//   - journal_mode=WAL, so a reader — an operator running
//     `sqlite3 <db> .dump` over their own data — takes a snapshot
//     instead of blocking every writer behind it.
//
//   - busy_timeout, so a writer that does meet a lock waits for it.
//     Without one SQLite gives up immediately; nothing above it
//     retries.
//
//   - _txlock=immediate, so every transaction takes the write lock at
//     BEGIN. A deferred transaction acquires it lazily on its first
//     write, and that upgrade returns SQLITE_BUSY *without* consulting
//     the busy handler, because SQLite cannot block a transaction that
//     may already hold a read snapshot. Such a COMMIT then fails while
//     the transaction stays open on the connection. A busy timeout
//     alone does not prevent this; BEGIN IMMEDIATE does, by putting
//     the wait somewhere the handler applies.
//
// Note what is absent: `cache=shared`. Under a shared cache an
// in-process conflict is reported as SQLITE_LOCKED rather than
// SQLITE_BUSY, and the busy handler does not retry SQLITE_LOCKED — so
// leaving it in would have defeated the busy timeout for exactly the
// contention this service generates. Dropping it is part of the fix,
// not housekeeping.
//
// synchronous is deliberately left at SQLite's default of FULL: this
// is a webhook receiver whose one promise is that an event it answered
// 200 for is durable.
// The order of the _pragma parameters is load-bearing.
// modernc.org/sqlite executes them in the order they appear, on every
// new connection, before the connection is handed to the pool. Setting
// journal_mode first means that pragma itself runs with no busy
// handler installed: the pool opens connections lazily, so the moment
// a new one is created is a moment the database is under load, and
// PRAGMA journal_mode takes a lock. It would fail immediately with
// SQLITE_BUSY and fail the query that caused the connection to be
// opened. busy_timeout is therefore set first, so every pragma after
// it — and the whole life of the connection — is covered.
func SQLiteDSN(path, mode string) string {
	q := url.Values{}
	q.Set("mode", mode)
	q.Set("_txlock", "immediate")
	q.Add(
		"_pragma",
		fmt.Sprintf(
			"busy_timeout(%d)",
			SQLiteBusyTimeout.Milliseconds(),
		),
	)
	q.Add("_pragma", "journal_mode(WAL)")

	return "file:" + path + "?" + q.Encode()
}

// OpenSQLite opens the SQLite file at path with the service's
// durability settings and pool bounds applied. mode is the SQLite URI
// open mode ("rwc" or "rw").
//
// The file and its WAL sidecars are settled at SQLiteFilePerm before
// the driver sees the path; see reserveSQLiteFile.
//
// The handle is returned rather than a *gorm.DB because the callers
// wrap it in gorm themselves with their own logger.
func OpenSQLite(path, mode string) (*sql.DB, error) {
	err := reserveSQLiteFile(path, mode == SQLiteModeCreate)
	if err != nil {
		return nil, err
	}

	sqlDB, err := sql.Open("sqlite", SQLiteDSN(path, mode))
	if err != nil {
		return nil, fmt.Errorf(
			"opening sqlite database %s: %w", path, err,
		)
	}

	sqlDB.SetMaxOpenConns(sqliteMaxOpenConns)
	sqlDB.SetMaxIdleConns(sqliteMaxIdleConns)
	sqlDB.SetConnMaxLifetime(sqliteConnMaxLifetime)
	sqlDB.SetConnMaxIdleTime(sqliteConnMaxIdleTime)

	return sqlDB, nil
}