Checkpoint without waiting on readers and keep the WAL file small (closes #102)

The 5-second checkpoint ran in TRUNCATE mode, which waits up to the
5-second busy timeout for every open read while holding the database
lock every batch write needs. It now runs in PASSIVE mode, which never
waits. Because PASSIVE alone never lets the WAL start again while reads
overlap, a WAL file over 64 MiB gets a TRUNCATE checkpoint whose wait
for reads is capped at 500 ms and logged. A checkpoint that leaves part
of the WAL uncopied returns ErrCheckpointIncomplete, which the
maintainer logs as incomplete instead of completed. The writing
connection's journal_size_limit is 64 MiB.

Model: opus-5-5

internal/database/database.go

@@ -81,6 +81,21 @@ const (
 	sqliteHardHeapLimitBytes = 2147483648
 )
 
+// Settings that keep the WAL file small (see Checkpoint).
+const (
+	// sqliteWALSizeLimitBytes (64 MiB) is the WAL file size past which
+	// Checkpoint truncates the file. It is also the journal_size_limit of the
+	// connection kept for writing: when SQLite starts the WAL again from the
+	// beginning, it cuts a larger WAL file down to this size.
+	sqliteWALSizeLimitBytes = 67108864
+	// truncateBusyTimeoutMs is the busy timeout of the TRUNCATE checkpoint:
+	// the longest it waits for open reads, and so the longest it keeps batch
+	// writes waiting beyond the copy itself.
+	truncateBusyTimeoutMs = 500
+	// walFileSuffix ends the name of the WAL file, next to the database file.
+	walFileSuffix = "-wal"
+)
+
 // Settings of the ANALYZE that Analyze runs.
 const (
 	// analysisLimit is about how many rows of each index ANALYZE reads
@@ -112,6 +127,11 @@ var (
 	ErrNoRoute = errors.New("no route found")
 	// ErrNoStaleASN is returned when no ASN needs WHOIS refresh
 	ErrNoStaleASN = errors.New("no stale ASN found")
+	// ErrCheckpointIncomplete is returned by Checkpoint when it left part of
+	// the WAL uncopied, or left a WAL file over sqliteWALSizeLimitBytes
+	// untruncated, because reads still needed the WAL or SQLite reported the
+	// checkpoint busy. A later checkpoint finishes the work.
+	ErrCheckpointIncomplete = errors.New("WAL checkpoint incomplete")
 
 	errInvalidASN       = errors.New("invalid ASN")
 	errSameASNInPeering = errors.New("cannot create peering with same ASN")
@@ -194,6 +214,13 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 			err)
 	}
 
+	_, err = writer.ExecContext(ctx,
+		fmt.Sprintf("PRAGMA journal_size_limit=%d", sqliteWALSizeLimitBytes))
+	if err != nil {
+		return nil, fmt.Errorf(
+			"failed to set the writing connection's WAL size limit: %w", err)
+	}
+
 	database := &Database{
 		db: db, writer: writer, logger: logger, path: dbPath,
 		counts: &liveCounts{},
@@ -2255,9 +2282,25 @@ func (d *Database) Analyze(ctx context.Context) error {
 	return nil
 }
 
-// Checkpoint runs a WAL checkpoint to transfer data from the WAL to the main database.
-// Uses TRUNCATE mode which blocks writers briefly but ensures complete checkpoint,
-// keeping the WAL small for fast read performance. It holds the database lock.
+// Checkpoint copies the WAL into the database file and keeps the WAL file
+// small. It holds the database lock, so batch writes wait while it runs.
+//
+// It runs a PASSIVE checkpoint, which never waits: it copies what no open read
+// still needs. That alone does not keep the file small: SQLite starts the WAL
+// again from the beginning only once a checkpoint has copied all of it and no
+// read that began before is still open, which never happens while reads keep
+// overlapping. So when the file is over sqliteWALSizeLimitBytes, Checkpoint
+// runs a TRUNCATE checkpoint instead, which waits for the reads that still need
+// the WAL, copies all of it and empties the file. That checkpoint waits at most
+// truncateBusyTimeoutMs, and it is logged with how long it took.
+//
+// So while reads overlap, the WAL file stays under sqliteWALSizeLimitBytes
+// plus what batch writes add between two checkpoints, as long as no read stays
+// open longer than that wait. A longer read makes the TRUNCATE checkpoint give
+// up, and the file grows until a later one empties it.
+//
+// It returns ErrCheckpointIncomplete when it leaves part of the WAL uncopied,
+// or a file over the limit untruncated.
 func (d *Database) Checkpoint(ctx context.Context) error {
 	d.lock("Checkpoint")
 	defer d.unlock()
@@ -2267,11 +2310,32 @@ func (d *Database) Checkpoint(ctx context.Context) error {
 		return fmt.Errorf("failed to checkpoint WAL: %w", err)
 	}
 
-	_, err = d.writer.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
+	wal, err := os.Stat(d.path + walFileSuffix)
+	if err != nil {
+		return fmt.Errorf("failed to read the WAL file's size: %w", err)
+	}
+
+	if wal.Size() > sqliteWALSizeLimitBytes {
+		return d.truncateWAL(ctx, wal.Size())
+	}
+
+	var busy, walPages, copiedPages int
+
+	err = d.writer.QueryRowContext(ctx, "PRAGMA wal_checkpoint(PASSIVE)").
+		Scan(&busy, &walPages, &copiedPages)
 	if err != nil {
 		return fmt.Errorf("failed to checkpoint WAL: %w", err)
 	}
 
+	if busy != 0 {
+		return fmt.Errorf("%w: SQLite reported it busy", ErrCheckpointIncomplete)
+	}
+
+	if copiedPages < walPages {
+		return fmt.Errorf("%w: copied %d of %d WAL pages",
+			ErrCheckpointIncomplete, copiedPages, walPages)
+	}
+
 	return nil
 }
 
@@ -2287,6 +2351,50 @@ func (d *Database) Ping(ctx context.Context) error {
 	return nil
 }
 
+// truncateWAL runs the TRUNCATE checkpoint of Checkpoint on a WAL file of
+// walBytes, with the writing connection's busy timeout lowered to
+// truncateBusyTimeoutMs for it, and logs how long it took. The caller holds the
+// database lock.
+func (d *Database) truncateWAL(ctx context.Context, walBytes int64) error {
+	_, err := d.writer.ExecContext(ctx,
+		fmt.Sprintf("PRAGMA busy_timeout=%d", truncateBusyTimeoutMs))
+	if err != nil {
+		return fmt.Errorf("failed to lower the busy timeout to truncate WAL: %w",
+			err)
+	}
+
+	started := time.Now()
+
+	var busy, walPages, copiedPages int
+
+	checkpointErr := d.writer.QueryRowContext(ctx,
+		"PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walPages, &copiedPages)
+	took := time.Since(started)
+
+	// Restored even when ctx has ended, so that no batch write runs with the
+	// short timeout.
+	_, restoreErr := d.writer.ExecContext(context.WithoutCancel(ctx),
+		fmt.Sprintf("PRAGMA busy_timeout=%d", sqliteBusyTimeoutMs))
+	if checkpointErr != nil || restoreErr != nil {
+		return fmt.Errorf("failed to truncate WAL: %w",
+			errors.Join(checkpointErr, restoreErr))
+	}
+
+	if busy != 0 {
+		d.logger.Warn("WAL file over its size limit not truncated",
+			"wal_bytes", walBytes, "wal_pages", walPages,
+			"copied_pages", copiedPages, "duration", took)
+
+		return fmt.Errorf("%w: WAL file of %d bytes not truncated",
+			ErrCheckpointIncomplete, walBytes)
+	}
+
+	d.logger.Info("WAL file over its size limit truncated",
+		"wal_bytes", walBytes, "duration", took)
+
+	return nil
+}
+
 // The debug messages lock logs, with the operation's name, when it starts to
 // wait for the database mutex and when it holds it.
 const (