Log the start-up checkpoint's figures under their own names (closes #117)

Initialize scanned the row of PRAGMA wal_checkpoint(TRUNCATE), which is
the busy flag, the frames in the WAL and the frames copied, into
wal_pages, checkpointed and moved; and a completed TRUNCATE reads 0 for
both counts. The line now logs busy, wal_frames and checkpointed_frames,
plus wal_bytes, the WAL file's size, which New reads before it opens the
database because the open writes a frame to the WAL. The line is a
warning when busy is 1. Tests open a database a second time while the
first stays open, with and without a read open on the first, and check
the line's level and figures.

Model: opus-5-5

README.md

@@ -5,6 +5,22 @@ table in real time: it streams BGP UPDATE messages from the RIPE RIS Live
 service, maintains a live routing table in SQLite, and provides HTTP APIs for
 querying routing information.
 
+## Rationale
+
+RIPE RIS Live streams the BGP updates that RIPE's route collectors receive, as
+they happen, from the moment a client connects. It reports changes, not the
+routing table they add up to, so it cannot say which AS an IP address is routed
+to now or which routes a prefix has. RouteWatch applies the feed's updates to
+live routing tables in its own SQLite database and answers IP-to-AS lookups and
+prefix pages from them.
+
+The feed sends no table when a client connects, so these tables hold only the
+routes the feed has announced since RouteWatch first started: an IP-to-AS lookup
+finds nothing for an address whose routes have not been announced since then.
+The tables also miss the updates RouteWatch drops when it falls behind under
+load and those sent while it is stopped, so a lookup or a prefix page can show a
+route the feed has withdrawn.
+
 ## Features
 
 - Real-time streaming of BGP updates from RIPE RIS Live

internal/database/database.go

@@ -169,6 +169,20 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 		return nil, fmt.Errorf("failed to create database directory: %w", err)
 	}
 
+	// The size of the WAL file the previous run left, which Initialize logs
+	// with the start-up checkpoint. It is read before the database is opened,
+	// because opening it writes a frame to the WAL. There is no WAL file when
+	// the database is new or the last run closed the database, which deletes
+	// it.
+	var walBytes int64
+
+	wal, err := os.Stat(dbPath + walFileSuffix)
+	if err == nil {
+		walBytes = wal.Size()
+	} else if !errors.Is(err, os.ErrNotExist) {
+		logger.Warn("Failed to read the WAL file's size on startup", "error", err)
+	}
+
 	// Per-connection SQLite settings go in the DSN so every pooled connection
 	// gets them, not just the one that runs the Initialize pragmas. _txlock=
 	// immediate makes every transaction take the write lock at BEGIN. Without it
@@ -229,7 +243,7 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 		counts: &liveCounts{},
 	}
 
-	err = database.Initialize()
+	err = database.Initialize(walBytes)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize database: %w", err)
 	}
@@ -251,8 +265,10 @@ const (
 	startupCheckpointIncompleteMessage = "WAL checkpoint on startup incomplete"
 )
 
-// Initialize creates the database schema if it doesn't exist.
-func (d *Database) Initialize() error {
+// Initialize creates the database schema if it doesn't exist. walBytes is the
+// size of the WAL file the previous run left, which New reads before it opens
+// the database.
+func (d *Database) Initialize(walBytes int64) error {
 	// Set SQLite pragmas for performance. Per-connection settings (cache_size,
 	// synchronous, busy_timeout, journal_mode) live in the DSN, and New sets the
 	// writing connection's cache_size; temp_store is left at its default so
@@ -274,21 +290,10 @@ func (d *Database) Initialize() error {
 
 	// The start-up checkpoint copies the WAL a previous run left into the
 	// database file and empties the WAL file; its frame counts then read 0,
-	// so the WAL file's size is read first. There is no WAL file when the
-	// database is new or the last run closed the database, which deletes it.
-	var walBytes int64
-
-	wal, err := os.Stat(d.path + walFileSuffix)
-	if err == nil {
-		walBytes = wal.Size()
-	} else if !errors.Is(err, os.ErrNotExist) {
-		d.logger.Warn("Failed to read the WAL file's size on startup",
-			"error", err)
-	}
-
+	// so walBytes is logged with them.
 	var busy, walFrames, checkpointedFrames int
 
-	err = d.writer.QueryRowContext(context.Background(),
+	err := d.writer.QueryRowContext(context.Background(),
 		"PRAGMA wal_checkpoint(TRUNCATE)",
 	).Scan(&busy, &walFrames, &checkpointedFrames)
 

internal/database/database_internal_test.go

@@ -1057,10 +1057,9 @@ func walFileSize(t *testing.T, db *Database) int64 {
 // TestStartupCheckpointLogsItsFigures opens a new database, writes a route,
 // and opens the same file again while the first stays open, so that its WAL
 // file stays. The first open finds no WAL file: it logs a size of 0 and no
-// warning. The second logs the size of the WAL file it found, at least what
-// the first left (opening the file writes to the WAL before the size is
-// read), and the checkpoint's busy flag and frame counts, each under its own
-// name; the counts read 0 once the checkpoint has emptied the file.
+// warning. The second logs the size of the WAL file the first left, and the
+// checkpoint's busy flag and frame counts, each under its own name; the
+// counts read 0 once the checkpoint has emptied the file.
 // Initialize logged the busy flag as wal_pages and the counts under each
 // other's names (https://git.eeqj.de/sneak/routewatch/issues/117).
 func TestStartupCheckpointLogsItsFigures(t *testing.T) {
@@ -1103,10 +1102,9 @@ func TestStartupCheckpointLogsItsFigures(t *testing.T) {
 		}
 	}
 
-	loggedBytes, _ := line["wal_bytes"].(float64)
-	if loggedBytes < float64(walBytes) {
-		t.Errorf("start-up checkpoint logged wal_bytes = %v, want at least "+
-			"the %d bytes the first database left", line["wal_bytes"], walBytes)
+	if line["wal_bytes"] != float64(walBytes) {
+		t.Errorf("start-up checkpoint logged wal_bytes = %v, want the %d "+
+			"bytes the first database left", line["wal_bytes"], walBytes)
 	}
 }
 
@@ -1156,6 +1154,11 @@ func TestStartupCheckpointWarnsWhenIncomplete(t *testing.T) {
 	_, lines := openLogged(t, cfg)
 	line := findLine(t, lines, startupCheckpointIncompleteMessage)
 
+	if line[slog.LevelKey] != slog.LevelWarn.String() {
+		t.Errorf("start-up checkpoint logged at level %v, want %s",
+			line[slog.LevelKey], slog.LevelWarn)
+	}
+
 	if line["busy"] != 1.0 {
 		t.Errorf("start-up checkpoint logged busy = %v, want 1", line["busy"])
 	}