clawbot
281d162e27
check / check (push) Canceled after 0s
The home page gets cards for the newest and oldest kept route, the newest and oldest updated AS, the AS with the most prefixes, the ASes with a single prefix and fifteen well-known networks. Two more cards count the bogon announcements and the prefixes with several origin ASes. They link to the new pages /bogons and /multiple-origins, which list up to 1,000 entries each. The new package internal/bogon tells reserved prefixes and AS numbers, from the IANA special-purpose and address space registries. Table.Overview walks the table and lets the read lock go every 10,000 prefixes. The server works the cards out once a minute and serves that copy. A new index on asns(last_seen) finds the newest and oldest updated AS. Model: opus-5-5
1531 lines
42 KiB
Go
1531 lines
42 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
_ "embed"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"net/netip"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"slices"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
"git.eeqj.de/sneak/routewatch/pkg/asinfo"
|
|
_ "github.com/mattn/go-sqlite3" // CGO SQLite driver
|
|
)
|
|
|
|
// IMPORTANT: NO schema changes are to be made outside of schema.sql
|
|
// We do NOT support migrations. All schema changes MUST be made in schema.sql only.
|
|
//
|
|
//go:embed schema.sql
|
|
var dbSchema string
|
|
|
|
const (
|
|
dirPermissions = 0750 // rwxr-x---
|
|
ipVersionV4 = 4
|
|
ipVersionV6 = 6
|
|
|
|
// Names of the IP versions in error and log messages.
|
|
ipv4Family = "IPv4"
|
|
ipv6Family = "IPv6"
|
|
)
|
|
|
|
// SQLite memory tuning (README "Memory"). cache_size and busy_timeout go in the
|
|
// DSN so every pooled connection gets them; the connection kept for writing
|
|
// then sets its own cache size. The heap limits are process-wide and set once.
|
|
const (
|
|
// maxConns is the size of the connection pool: the connection kept for
|
|
// writing and 9 reading connections.
|
|
maxConns = 10
|
|
// sqliteReadCacheSizeKiB is the page cache of each reading connection;
|
|
// negative means KiB. -16384 = 16 MiB, at most 144 MiB for the 9. A reading
|
|
// connection's cache is emptied whenever another connection has written
|
|
// since its last read, many times a minute, so a large one is wasted.
|
|
sqliteReadCacheSizeKiB = -16384
|
|
// sqliteWriteCacheSizeKiB is the page cache of the connection kept for
|
|
// writing, -524288 = 512 MiB. It keeps its pages from one write to the
|
|
// next. The routes table of a full routing table takes about 450 MB, so
|
|
// the pages the writes of the table's changes touch stay in memory.
|
|
sqliteWriteCacheSizeKiB = -524288
|
|
// sqliteBusyTimeoutMs is how long a connection waits on a locked database.
|
|
sqliteBusyTimeoutMs = 5000
|
|
// sqliteSoftHeapLimitBytes (768 MiB) makes SQLite recycle its cache rather
|
|
// than allocate once its C heap passes this size. It is above the caches
|
|
// together, 656 MiB, so it does not shrink them.
|
|
sqliteSoftHeapLimitBytes = 805306368
|
|
// sqliteHardHeapLimitBytes (1 GiB) fails a statement with SQLITE_NOMEM
|
|
// instead of growing the C heap without bound.
|
|
sqliteHardHeapLimitBytes = 1073741824
|
|
)
|
|
|
|
// Settings that bound the WAL file's size once no read is open (see
|
|
// Checkpoint).
|
|
const (
|
|
// sqliteWALSizeLimitBytes (64 MiB) is the WAL file size past which
|
|
// Checkpoint runs a TRUNCATE checkpoint. It is also the
|
|
// journal_size_limit of the connection kept for writing: when SQLite
|
|
// starts the WAL again from the beginning, it cuts a larger WAL file down
|
|
// to this size.
|
|
sqliteWALSizeLimitBytes = 67108864
|
|
// truncateBusyTimeoutMs is the busy timeout of the TRUNCATE checkpoint:
|
|
// the longest it waits for open reads, and so the longest it keeps batch
|
|
// writes waiting beyond the copy itself.
|
|
truncateBusyTimeoutMs = 500
|
|
// walFileSuffix ends the name of the WAL file, next to the database file.
|
|
walFileSuffix = "-wal"
|
|
)
|
|
|
|
// Settings of the ANALYZE that Analyze runs.
|
|
const (
|
|
// analysisLimit is about how many rows of each index ANALYZE reads
|
|
// (PRAGMA analysis_limit); SQLite estimates the rest.
|
|
analysisLimit = 1000
|
|
// optimizeAllTables is the PRAGMA optimize mask that runs ANALYZE (0x2)
|
|
// on every table that has an index without statistics or whose size has
|
|
// changed tenfold since it was last analyzed, not only on the tables this
|
|
// connection has queried (0x10000).
|
|
optimizeAllTables = 0x10002
|
|
)
|
|
|
|
// Common errors
|
|
var (
|
|
// ErrNoStaleASN is returned when no ASN needs WHOIS refresh
|
|
ErrNoStaleASN = errors.New("no stale ASN found")
|
|
// ErrCheckpointIncomplete is returned by Checkpoint when it left part of
|
|
// the WAL uncopied, or left a WAL file over sqliteWALSizeLimitBytes
|
|
// untruncated, because reads still needed the WAL or SQLite reported the
|
|
// checkpoint busy. A later checkpoint that runs when no read is open
|
|
// finishes the work.
|
|
ErrCheckpointIncomplete = errors.New("WAL checkpoint incomplete")
|
|
|
|
errInvalidASN = errors.New("invalid ASN")
|
|
errSameASNInPeering = errors.New("cannot create peering with same ASN")
|
|
)
|
|
|
|
// Database manages the SQLite database connection and operations.
|
|
type Database struct {
|
|
db *sql.DB
|
|
// writer is the connection kept for writing, taken from db when the
|
|
// database opens. Every write uses it while holding mu.
|
|
writer *sql.Conn
|
|
logger *logger.Logger
|
|
path string
|
|
mu sync.Mutex
|
|
lockedAt time.Time
|
|
lockedBy string
|
|
counts *liveCounts
|
|
}
|
|
|
|
// New creates a new database connection and initializes the schema.
|
|
func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
|
|
dbPath := filepath.Join(cfg.GetStateDir(), "db.sqlite")
|
|
|
|
// Log database path
|
|
logger.Info("Opening database", "path", dbPath)
|
|
|
|
// Ensure directory exists
|
|
dir := filepath.Dir(dbPath)
|
|
|
|
err := os.MkdirAll(dir, dirPermissions)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create database directory: %w", err)
|
|
}
|
|
|
|
// The size of the WAL file the previous run left, which Initialize logs
|
|
// with the start-up checkpoint. It is read before the database is opened,
|
|
// because opening it writes a frame to the WAL. There is no WAL file when
|
|
// the database is new or the last run closed the database, which deletes
|
|
// it.
|
|
var walBytes int64
|
|
|
|
wal, err := os.Stat(dbPath + walFileSuffix)
|
|
if err == nil {
|
|
walBytes = wal.Size()
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
logger.Warn("Failed to read the WAL file's size on startup", "error", err)
|
|
}
|
|
|
|
// Per-connection SQLite settings go in the DSN so every pooled connection
|
|
// gets them, not just the one that runs the Initialize pragmas. _txlock=
|
|
// immediate makes every transaction take the write lock at BEGIN. Without it
|
|
// a transaction that reads before writing starts as a reader and, when it
|
|
// then writes while another connection holds the write lock, fails at once
|
|
// with "database is locked" without waiting for _busy_timeout.
|
|
// _auto_vacuum=incremental lets Vacuum return free pages to the filesystem.
|
|
// SQLite only accepts it before the database file is first written, and the
|
|
// switch to WAL writes it, so it must be here: the driver applies it on open,
|
|
// before _journal_mode. On an existing file it changes nothing.
|
|
dsn := fmt.Sprintf(
|
|
"file:%s?_cache_size=%d&_synchronous=OFF&_busy_timeout=%d"+
|
|
"&_auto_vacuum=incremental&_journal_mode=WAL&_txlock=immediate",
|
|
dbPath,
|
|
sqliteReadCacheSizeKiB,
|
|
sqliteBusyTimeoutMs,
|
|
)
|
|
|
|
db, err := sql.Open("sqlite3", dsn)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open database: %w", err)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
|
|
err = db.PingContext(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to ping database: %w", err)
|
|
}
|
|
|
|
// Set connection pool parameters
|
|
// Multiple connections allow concurrent reads while writes are serialized
|
|
db.SetMaxOpenConns(maxConns)
|
|
db.SetMaxIdleConns(maxConns)
|
|
db.SetConnMaxLifetime(0)
|
|
|
|
writer, err := db.Conn(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open the writing connection: %w", err)
|
|
}
|
|
|
|
_, err = writer.ExecContext(ctx,
|
|
fmt.Sprintf("PRAGMA cache_size=%d", sqliteWriteCacheSizeKiB))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to set the writing connection's cache: %w",
|
|
err)
|
|
}
|
|
|
|
_, err = writer.ExecContext(ctx,
|
|
fmt.Sprintf("PRAGMA journal_size_limit=%d", sqliteWALSizeLimitBytes))
|
|
if err != nil {
|
|
return nil, fmt.Errorf(
|
|
"failed to set the writing connection's WAL size limit: %w", err)
|
|
}
|
|
|
|
database := &Database{
|
|
db: db, writer: writer, logger: logger, path: dbPath,
|
|
counts: &liveCounts{},
|
|
}
|
|
|
|
err = database.Initialize(walBytes)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to initialize database: %w", err)
|
|
}
|
|
|
|
// Seed the in-memory statistics counters from the tables once, before the
|
|
// streamer starts writing. From here on every write keeps them current, so
|
|
// the stats endpoints never scan the tables to report counts.
|
|
err = database.seedCounts(context.Background())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to seed statistics counters: %w", err)
|
|
}
|
|
|
|
return database, nil
|
|
}
|
|
|
|
// The messages Initialize logs the start-up WAL checkpoint with.
|
|
const (
|
|
startupCheckpointMessage = "WAL checkpoint on startup"
|
|
startupCheckpointIncompleteMessage = "WAL checkpoint on startup incomplete"
|
|
)
|
|
|
|
// Initialize creates the database schema if it doesn't exist. walBytes is the
|
|
// size of the WAL file the previous run left, which New reads before it opens
|
|
// the database.
|
|
func (d *Database) Initialize(walBytes int64) error {
|
|
// Set SQLite pragmas for performance. Per-connection settings (cache_size,
|
|
// synchronous, busy_timeout, journal_mode) live in the DSN, and New sets the
|
|
// writing connection's cache_size; temp_store is left at its default so
|
|
// DISTINCT temp B-trees spill to disk instead of C heap. The heap limits
|
|
// below are process-wide, so setting them once here is enough for the whole
|
|
// pool.
|
|
pragmas := []string{
|
|
"PRAGMA journal_mode=WAL", // Write-Ahead Logging
|
|
fmt.Sprintf("PRAGMA soft_heap_limit=%d", sqliteSoftHeapLimitBytes),
|
|
fmt.Sprintf("PRAGMA hard_heap_limit=%d", sqliteHardHeapLimitBytes),
|
|
}
|
|
|
|
for _, pragma := range pragmas {
|
|
err := d.exec(pragma)
|
|
if err != nil {
|
|
d.logger.Warn("Failed to set pragma", "pragma", pragma, "error", err)
|
|
}
|
|
}
|
|
|
|
// The start-up checkpoint copies the WAL a previous run left into the
|
|
// database file and empties the WAL file; its frame counts then read 0,
|
|
// so walBytes is logged with them.
|
|
var busy, walFrames, checkpointedFrames int
|
|
|
|
err := d.writer.QueryRowContext(context.Background(),
|
|
"PRAGMA wal_checkpoint(TRUNCATE)",
|
|
).Scan(&busy, &walFrames, &checkpointedFrames)
|
|
|
|
switch {
|
|
case err != nil:
|
|
d.logger.Warn("Failed to checkpoint WAL on startup", "error", err)
|
|
case busy != 0:
|
|
// Another process using the database kept the checkpoint from
|
|
// copying or emptying all of the WAL.
|
|
d.logger.Warn(startupCheckpointIncompleteMessage,
|
|
"wal_bytes", walBytes,
|
|
"busy", busy,
|
|
"wal_frames", walFrames,
|
|
"checkpointed_frames", checkpointedFrames,
|
|
)
|
|
default:
|
|
d.logger.Info(startupCheckpointMessage,
|
|
"wal_bytes", walBytes,
|
|
"busy", busy,
|
|
"wal_frames", walFrames,
|
|
"checkpointed_frames", checkpointedFrames,
|
|
)
|
|
}
|
|
|
|
err = d.exec(dbSchema)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Close closes the writing connection and then the connection pool.
|
|
func (d *Database) Close() error {
|
|
return errors.Join(d.writer.Close(), d.db.Close())
|
|
}
|
|
|
|
// insertPrefixesInTable inserts the prefixes, all of one IP version, that the
|
|
// prefix table of that version does not hold yet, each first seen at its time
|
|
// in prefixes. A prefix already stored is left as it is. It returns how many it
|
|
// inserted.
|
|
func insertPrefixesInTable(
|
|
ctx context.Context, tx *loggingTx, prefixes map[string]time.Time,
|
|
ipVersion int,
|
|
) (int, error) {
|
|
if len(prefixes) == 0 {
|
|
return 0, nil
|
|
}
|
|
|
|
family := ipv4Family
|
|
insertSQL := "INSERT INTO prefixes_v4 (prefix, first_seen) VALUES (?, ?) " +
|
|
"ON CONFLICT (prefix) DO NOTHING"
|
|
|
|
if ipVersion == ipVersionV6 {
|
|
family = ipv6Family
|
|
insertSQL = "INSERT INTO prefixes_v6 (prefix, first_seen) VALUES (?, ?) " +
|
|
"ON CONFLICT (prefix) DO NOTHING"
|
|
}
|
|
|
|
insertStmt, err := tx.PrepareContext(ctx, insertSQL)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to prepare %s insert statement: %w",
|
|
family, err)
|
|
}
|
|
defer func() { _ = insertStmt.Close() }()
|
|
|
|
var newPrefixes int
|
|
|
|
// The rows are written in key order, so that consecutive writes land on
|
|
// the same index pages.
|
|
for _, prefix := range slices.Sorted(maps.Keys(prefixes)) {
|
|
res, err := insertStmt.ExecContext(ctx, prefix, prefixes[prefix])
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to insert prefix %s: %w", prefix, err)
|
|
}
|
|
|
|
// One row for a new prefix, none for one already stored.
|
|
inserted, err := res.RowsAffected()
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to count inserted prefix %s: %w",
|
|
prefix, err)
|
|
}
|
|
|
|
if inserted > 0 {
|
|
newPrefixes++
|
|
}
|
|
}
|
|
|
|
return newPrefixes, nil
|
|
}
|
|
|
|
// UpdatePrefixesBatch stores, in a single transaction, the prefixes that are
|
|
// not stored yet, each first seen at its time in prefixes: the IPv4 prefixes
|
|
// first and then the IPv6 prefixes. A prefix that does not parse is not stored.
|
|
func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
|
if len(prefixes) == 0 {
|
|
return nil
|
|
}
|
|
|
|
d.lock("UpdatePrefixesBatch")
|
|
defer d.unlock()
|
|
|
|
tx, err := d.beginTx()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to begin transaction: %w", err)
|
|
}
|
|
defer d.rollback(tx)
|
|
|
|
prefixesV4 := make(map[string]time.Time)
|
|
prefixesV6 := make(map[string]time.Time)
|
|
|
|
for prefix, timestamp := range prefixes {
|
|
_, err := netip.ParsePrefix(prefix)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
if detectIPVersion(prefix) == ipVersionV4 {
|
|
prefixesV4[prefix] = timestamp
|
|
} else {
|
|
prefixesV6[prefix] = timestamp
|
|
}
|
|
}
|
|
|
|
ctx := context.Background()
|
|
|
|
newV4, err := insertPrefixesInTable(ctx, tx, prefixesV4, ipVersionV4)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
newV6, err := insertPrefixesInTable(ctx, tx, prefixesV6, ipVersionV6)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to commit transaction: %w", err)
|
|
}
|
|
|
|
d.counts.addPrefixes(newV4, newV6)
|
|
|
|
return nil
|
|
}
|
|
|
|
// These statements update a stored AS, peering or peer. Each keeps the earlier
|
|
// of the stored and the new first_seen, and the later of the stored and the new
|
|
// last_seen. Some collectors send updates days old while they catch up on a
|
|
// backlog. MIN and MAX compare the times as text, which is time order here: the
|
|
// driver writes every time in one format, and every time is UTC.
|
|
const (
|
|
updateASNSQL = `UPDATE asns
|
|
SET first_seen = MIN(first_seen, ?), last_seen = MAX(last_seen, ?)
|
|
WHERE asn = ?`
|
|
|
|
updatePeeringSQL = `UPDATE peerings
|
|
SET first_seen = MIN(first_seen, ?), last_seen = MAX(last_seen, ?)
|
|
WHERE as_a = ? AND as_b = ?`
|
|
|
|
updatePeerSQL = `UPDATE bgp_peers
|
|
SET peer_asn = ?, last_message_type = ?,
|
|
first_seen = MIN(first_seen, ?), last_seen = MAX(last_seen, ?)
|
|
WHERE peer_ip = ?`
|
|
)
|
|
|
|
// newASN returns a new ASN record first seen at seen.First and last seen at
|
|
// seen.Last, with the handle and description from the AS info lookup when it
|
|
// knows the number.
|
|
func newASN(number int, seen SeenTimes) ASN {
|
|
asn := ASN{
|
|
ASN: number,
|
|
FirstSeen: seen.First,
|
|
LastSeen: seen.Last,
|
|
}
|
|
|
|
// Look up ASN info
|
|
if info, ok := asinfo.Get(number); ok {
|
|
asn.Handle = info.Handle
|
|
asn.Description = info.Description
|
|
}
|
|
|
|
return asn
|
|
}
|
|
|
|
// upsertASN stores the seen times of ASN number with the given statements,
|
|
// inserting the ASN when it is new. It reports whether it inserted it.
|
|
func upsertASN(
|
|
ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt,
|
|
number int, seen SeenTimes,
|
|
) (bool, error) {
|
|
var (
|
|
asn ASN
|
|
handle, description sql.NullString
|
|
)
|
|
|
|
err := selectStmt.QueryRowContext(ctx, number).Scan(&asn.ASN, &handle,
|
|
&description, &asn.FirstSeen, &asn.LastSeen)
|
|
if err == nil {
|
|
// ASN exists, update its seen times
|
|
_, err = updateStmt.ExecContext(ctx, seen.First, seen.Last, number)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to update ASN %d: %w", number, err)
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
if !errors.Is(err, sql.ErrNoRows) {
|
|
return false, fmt.Errorf("failed to query ASN %d: %w", number, err)
|
|
}
|
|
|
|
// ASN doesn't exist, create it
|
|
asn = newASN(number, seen)
|
|
|
|
_, err = insertStmt.ExecContext(ctx, asn.ASN, asn.Handle,
|
|
asn.Description, asn.FirstSeen, asn.LastSeen)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to insert ASN %d: %w", number, err)
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction
|
|
func (d *Database) GetOrCreateASNBatch(asns map[int]SeenTimes) error {
|
|
if len(asns) == 0 {
|
|
return nil
|
|
}
|
|
|
|
d.lock("GetOrCreateASNBatch")
|
|
defer d.unlock()
|
|
|
|
ctx := context.Background()
|
|
|
|
tx, err := d.beginTx()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to begin transaction: %w", err)
|
|
}
|
|
defer d.rollback(tx)
|
|
|
|
// Prepare statements
|
|
selectStmt, err := tx.PrepareContext(ctx,
|
|
"SELECT asn, handle, description, first_seen, last_seen "+
|
|
"FROM asns WHERE asn = ?")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to prepare select statement: %w", err)
|
|
}
|
|
defer func() { _ = selectStmt.Close() }()
|
|
|
|
updateStmt, err := tx.PrepareContext(ctx, updateASNSQL)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to prepare update statement: %w", err)
|
|
}
|
|
defer func() { _ = updateStmt.Close() }()
|
|
|
|
insertStmt, err := tx.PrepareContext(ctx,
|
|
"INSERT INTO asns (asn, handle, description, first_seen, last_seen) "+
|
|
"VALUES (?, ?, ?, ?, ?)")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to prepare insert statement: %w", err)
|
|
}
|
|
defer func() { _ = insertStmt.Close() }()
|
|
|
|
var newASNs int
|
|
|
|
for number, seen := range asns {
|
|
inserted, err := upsertASN(ctx, selectStmt, updateStmt, insertStmt,
|
|
number, seen)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if inserted {
|
|
newASNs++
|
|
}
|
|
}
|
|
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to commit transaction: %w", err)
|
|
}
|
|
|
|
d.counts.addASNs(newASNs)
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetOrCreateASN retrieves an existing ASN or creates a new one if it doesn't
|
|
// exist.
|
|
func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) {
|
|
d.lock("GetOrCreateASN")
|
|
defer d.unlock()
|
|
|
|
tx, err := d.beginTx()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer d.rollback(tx)
|
|
|
|
var (
|
|
asn ASN
|
|
handle, description sql.NullString
|
|
)
|
|
|
|
err = tx.QueryRow("SELECT asn, handle, description, first_seen, last_seen "+
|
|
"FROM asns WHERE asn = ?", number).
|
|
Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen)
|
|
if err == nil {
|
|
// ASN exists, update its seen times
|
|
asn.Handle = handle.String
|
|
asn.Description = description.String
|
|
|
|
_, err = tx.Exec(updateASNSQL, timestamp, timestamp, number)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Return the seen times the update stored.
|
|
seen := SeenTimes{First: asn.FirstSeen, Last: asn.LastSeen}
|
|
seen.Add(timestamp)
|
|
asn.FirstSeen, asn.LastSeen = seen.First, seen.Last
|
|
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
d.logger.Error("Failed to commit transaction for ASN update",
|
|
"asn", number, "error", err)
|
|
|
|
return nil, err
|
|
}
|
|
|
|
return &asn, nil
|
|
}
|
|
|
|
if !errors.Is(err, sql.ErrNoRows) {
|
|
return nil, err
|
|
}
|
|
|
|
// ASN doesn't exist, create it with ASN info lookup
|
|
asn = newASN(number, SeenTimes{First: timestamp, Last: timestamp})
|
|
|
|
_, err = tx.Exec("INSERT INTO asns "+
|
|
"(asn, handle, description, first_seen, last_seen) "+
|
|
"VALUES (?, ?, ?, ?, ?)",
|
|
asn.ASN, asn.Handle, asn.Description, asn.FirstSeen, asn.LastSeen)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
d.logger.Error("Failed to commit transaction for ASN creation",
|
|
"asn", number, "error", err)
|
|
|
|
return nil, err
|
|
}
|
|
|
|
d.counts.addASNs(1)
|
|
|
|
return &asn, nil
|
|
}
|
|
|
|
// GetOrCreatePrefix retrieves an existing prefix or creates a new one if it
|
|
// doesn't exist.
|
|
func (d *Database) GetOrCreatePrefix(
|
|
prefix string, timestamp time.Time) (*Prefix, error) {
|
|
d.lock("GetOrCreatePrefix")
|
|
defer d.unlock()
|
|
|
|
tx, err := d.beginTx()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer d.rollback(tx)
|
|
|
|
// Determine table based on IP version
|
|
ipVersion := detectIPVersion(prefix)
|
|
|
|
tableName := "prefixes_v4"
|
|
if ipVersion == ipVersionV6 {
|
|
tableName = "prefixes_v6"
|
|
}
|
|
|
|
var p Prefix
|
|
|
|
query := fmt.Sprintf("SELECT prefix, first_seen FROM %s WHERE prefix = ?",
|
|
tableName)
|
|
|
|
err = tx.QueryRow(query, prefix).Scan(&p.Prefix, &p.FirstSeen)
|
|
if err == nil {
|
|
// Prefix exists; the deferred rollback ends the read.
|
|
p.IPVersion = ipVersion
|
|
|
|
return &p, nil
|
|
}
|
|
|
|
if !errors.Is(err, sql.ErrNoRows) {
|
|
return nil, err
|
|
}
|
|
|
|
// Prefix doesn't exist, create it
|
|
p = Prefix{Prefix: prefix, IPVersion: ipVersion, FirstSeen: timestamp}
|
|
insertQuery := fmt.Sprintf("INSERT INTO %s "+
|
|
"(prefix, first_seen) VALUES (?, ?)", tableName)
|
|
|
|
_, err = tx.Exec(insertQuery, p.Prefix, p.FirstSeen)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
d.logger.Error("Failed to commit transaction for prefix creation",
|
|
"prefix", prefix, "error", err)
|
|
|
|
return nil, err
|
|
}
|
|
|
|
if ipVersion == ipVersionV4 {
|
|
d.counts.addPrefixes(1, 0)
|
|
} else {
|
|
d.counts.addPrefixes(0, 1)
|
|
}
|
|
|
|
return &p, nil
|
|
}
|
|
|
|
// RecordAnnouncement inserts a new BGP announcement or withdrawal into the
|
|
// database.
|
|
func (d *Database) RecordAnnouncement(announcement *Announcement) error {
|
|
d.lock("RecordAnnouncement")
|
|
defer d.unlock()
|
|
|
|
err := d.exec(`
|
|
INSERT INTO announcements (id, prefix_id, peer_asn, origin_asn, path,
|
|
next_hop, timestamp, is_withdrawal)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
announcement.ID.String(), announcement.PrefixID.String(),
|
|
announcement.PeerASN, announcement.OriginASN,
|
|
announcement.Path, announcement.NextHop, announcement.Timestamp,
|
|
announcement.IsWithdrawal)
|
|
|
|
return err
|
|
}
|
|
|
|
// RecordPeering records a peering relationship between two ASNs.
|
|
func (d *Database) RecordPeering(asA, asB int, seen SeenTimes) error {
|
|
// Validate ASNs
|
|
if asA <= 0 || asB <= 0 {
|
|
return fmt.Errorf("%w: asA=%d, asB=%d", errInvalidASN, asA, asB)
|
|
}
|
|
|
|
if asA == asB {
|
|
return fmt.Errorf("%w: %d", errSameASNInPeering, asA)
|
|
}
|
|
|
|
// Normalize: ensure asA < asB
|
|
if asA > asB {
|
|
asA, asB = asB, asA
|
|
}
|
|
|
|
d.lock("RecordPeering")
|
|
defer d.unlock()
|
|
|
|
tx, err := d.beginTx()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer d.rollback(tx)
|
|
|
|
var exists bool
|
|
|
|
err = tx.QueryRow(
|
|
"SELECT EXISTS(SELECT 1 FROM peerings WHERE as_a = ? AND as_b = ?)",
|
|
asA, asB).Scan(&exists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if exists {
|
|
_, err = tx.Exec(updatePeeringSQL, seen.First, seen.Last, asA, asB)
|
|
} else {
|
|
_, err = tx.Exec(`
|
|
INSERT INTO peerings (id, as_a, as_b, first_seen, last_seen)
|
|
VALUES (?, ?, ?, ?, ?)`,
|
|
generateUUID().String(), asA, asB, seen.First, seen.Last)
|
|
}
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
d.logger.Error("Failed to commit transaction for peering",
|
|
"as_a", asA,
|
|
"as_b", asB,
|
|
"error", err,
|
|
)
|
|
|
|
return err
|
|
}
|
|
|
|
if !exists {
|
|
d.counts.addPeerings(1)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// UpdatePeerBatch updates or creates multiple BGP peer records in a single transaction
|
|
func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error {
|
|
if len(peers) == 0 {
|
|
return nil
|
|
}
|
|
|
|
d.lock("UpdatePeerBatch")
|
|
defer d.unlock()
|
|
|
|
ctx := context.Background()
|
|
|
|
tx, err := d.beginTx()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to begin transaction: %w", err)
|
|
}
|
|
defer d.rollback(tx)
|
|
|
|
// Prepare statements
|
|
checkStmt, err := tx.PrepareContext(ctx,
|
|
"SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to prepare check statement: %w", err)
|
|
}
|
|
defer func() { _ = checkStmt.Close() }()
|
|
|
|
updateStmt, err := tx.PrepareContext(ctx, updatePeerSQL)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to prepare update statement: %w", err)
|
|
}
|
|
defer func() { _ = updateStmt.Close() }()
|
|
|
|
insertStmt, err := tx.PrepareContext(ctx,
|
|
"INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+
|
|
"last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to prepare insert statement: %w", err)
|
|
}
|
|
defer func() { _ = insertStmt.Close() }()
|
|
|
|
var newPeers int
|
|
|
|
for _, update := range peers {
|
|
var exists bool
|
|
|
|
err = checkStmt.QueryRowContext(ctx, update.PeerIP).Scan(&exists)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to check peer %s: %w", update.PeerIP, err)
|
|
}
|
|
|
|
if exists {
|
|
_, err = updateStmt.ExecContext(ctx, update.PeerASN,
|
|
update.MessageType, update.Seen.First, update.Seen.Last,
|
|
update.PeerIP)
|
|
} else {
|
|
_, err = insertStmt.ExecContext(ctx,
|
|
generateUUID().String(), update.PeerIP, update.PeerASN,
|
|
update.Seen.First, update.Seen.Last, update.MessageType)
|
|
}
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("failed to update peer %s: %w", update.PeerIP, err)
|
|
}
|
|
|
|
if !exists {
|
|
newPeers++
|
|
}
|
|
}
|
|
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to commit transaction: %w", err)
|
|
}
|
|
|
|
d.counts.addPeers(newPeers)
|
|
|
|
return nil
|
|
}
|
|
|
|
// UpdatePeer updates or creates a BGP peer record
|
|
func (d *Database) UpdatePeer(
|
|
peerIP string, peerASN int, messageType string, timestamp time.Time,
|
|
) error {
|
|
d.lock("UpdatePeer")
|
|
defer d.unlock()
|
|
|
|
tx, err := d.beginTx()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer d.rollback(tx)
|
|
|
|
var exists bool
|
|
|
|
err = tx.QueryRow(
|
|
"SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)", peerIP,
|
|
).Scan(&exists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if exists {
|
|
_, err = tx.Exec(updatePeerSQL,
|
|
peerASN, messageType, timestamp, timestamp, peerIP)
|
|
} else {
|
|
_, err = tx.Exec(
|
|
"INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+
|
|
"last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)",
|
|
generateUUID().String(), peerIP, peerASN, timestamp, timestamp,
|
|
messageType,
|
|
)
|
|
}
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
d.logger.Error("Failed to commit transaction for peer update",
|
|
"peer_ip", peerIP,
|
|
"peer_asn", peerASN,
|
|
"error", err,
|
|
)
|
|
|
|
return err
|
|
}
|
|
|
|
if !exists {
|
|
d.counts.addPeers(1)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetStats returns database statistics
|
|
func (d *Database) GetStats() (Stats, error) {
|
|
return d.GetStatsContext(context.Background())
|
|
}
|
|
|
|
// GetStatsContext returns database statistics with context support.
|
|
//
|
|
// The row counts (ASNs, prefixes, peerings, peers) come from the in-memory
|
|
// counters, seeded at startup and kept current on every write, and the file
|
|
// size from a stat(). The read touches no table.
|
|
func (d *Database) GetStatsContext(_ context.Context) (Stats, error) {
|
|
var stats Stats
|
|
|
|
// Row counts from memory, as a single consistent snapshot.
|
|
d.counts.fill(&stats)
|
|
|
|
// Database file size is a cheap stat() on the file.
|
|
fileInfo, err := os.Stat(d.path)
|
|
if err != nil {
|
|
d.logger.Warn("Failed to get database file size", "error", err)
|
|
} else {
|
|
stats.FileSizeBytes = fileInfo.Size()
|
|
}
|
|
|
|
return stats, nil
|
|
}
|
|
|
|
// GetAS returns the AS's handle, description and first and last seen times
|
|
// from the asns table. When the table has no record of the AS, its handle and
|
|
// description come from the AS info lookup, and it has no seen times. The bool
|
|
// says whether the asns table or the AS info lookup holds the AS.
|
|
func (d *Database) GetAS(ctx context.Context, asn int) (*ASN, bool, error) {
|
|
var (
|
|
asnInfo ASN
|
|
handle, description sql.NullString
|
|
)
|
|
|
|
err := d.db.QueryRowContext(ctx,
|
|
"SELECT asn, handle, description, first_seen, last_seen "+
|
|
"FROM asns WHERE asn = ?",
|
|
asn,
|
|
).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen,
|
|
&asnInfo.LastSeen)
|
|
|
|
switch {
|
|
case errors.Is(err, sql.ErrNoRows):
|
|
asnInfo.ASN = asn
|
|
|
|
info, ok := asinfo.Get(asn)
|
|
if ok {
|
|
asnInfo.Handle = info.Handle
|
|
asnInfo.Description = info.Description
|
|
}
|
|
|
|
return &asnInfo, ok, nil
|
|
case err != nil:
|
|
return nil, false, fmt.Errorf("failed to query AS: %w", err)
|
|
}
|
|
|
|
asnInfo.Handle = handle.String
|
|
asnInfo.Description = description.String
|
|
|
|
return &asnInfo, true, nil
|
|
}
|
|
|
|
// The queries of GetLastSeenASes. The index on last_seen answers each without
|
|
// reading the whole table.
|
|
const (
|
|
newestLastSeenASQuery = `
|
|
SELECT asn, handle, description, first_seen, last_seen
|
|
FROM asns ORDER BY last_seen DESC LIMIT 1`
|
|
oldestLastSeenASQuery = `
|
|
SELECT asn, handle, description, first_seen, last_seen
|
|
FROM asns ORDER BY last_seen ASC LIMIT 1`
|
|
)
|
|
|
|
// GetLastSeenASes returns two ASes of the asns table, with their handles,
|
|
// descriptions and seen times: the one last seen most recently, and the one
|
|
// last seen longest ago. Both are nil when the table holds no AS.
|
|
func (d *Database) GetLastSeenASes(ctx context.Context) (*ASN, *ASN, error) {
|
|
newest, found, err := d.readOneAS(ctx, newestLastSeenASQuery)
|
|
if err != nil || !found {
|
|
return nil, nil, err
|
|
}
|
|
|
|
oldest, _, err := d.readOneAS(ctx, oldestLastSeenASQuery)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
return newest, oldest, nil
|
|
}
|
|
|
|
// asWHOISQuery reads the handle, description and WHOIS details of one AS, and
|
|
// when the WHOIS details were fetched.
|
|
const asWHOISQuery = `
|
|
SELECT handle, description, as_name, org_name, org_id, address,
|
|
country_code, abuse_email, rir, whois_updated_at
|
|
FROM asns WHERE asn = ?`
|
|
|
|
// GetASWHOIS returns the handle, description and WHOIS details that the asns
|
|
// table holds for the AS, and when the WHOIS details were fetched. When the
|
|
// table has no record of the AS, it returns an ASN with only its number.
|
|
func (d *Database) GetASWHOIS(ctx context.Context, asn int) (*ASN, error) {
|
|
var (
|
|
handle, description, asName, orgName, orgID sql.NullString
|
|
address, countryCode, abuseEmail, rir sql.NullString
|
|
whoisUpdatedAt sql.NullTime
|
|
)
|
|
|
|
err := d.db.QueryRowContext(ctx, asWHOISQuery, asn).Scan(
|
|
&handle, &description, &asName, &orgName, &orgID, &address,
|
|
&countryCode, &abuseEmail, &rir, &whoisUpdatedAt,
|
|
)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return &ASN{ASN: asn}, nil
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to query AS: %w", err)
|
|
}
|
|
|
|
info := &ASN{
|
|
ASN: asn,
|
|
Handle: handle.String,
|
|
Description: description.String,
|
|
ASName: asName.String,
|
|
OrgName: orgName.String,
|
|
OrgID: orgID.String,
|
|
Address: address.String,
|
|
CountryCode: countryCode.String,
|
|
AbuseEmail: abuseEmail.String,
|
|
RIR: rir.String,
|
|
}
|
|
|
|
if whoisUpdatedAt.Valid {
|
|
info.WHOISUpdatedAt = &whoisUpdatedAt.Time
|
|
}
|
|
|
|
return info, nil
|
|
}
|
|
|
|
// The queries GetPrefixFirstSeen reads an IPv4 and an IPv6 prefix's first-seen
|
|
// time with.
|
|
const (
|
|
prefixFirstSeenV4SQL = `SELECT first_seen FROM prefixes_v4 WHERE prefix = ?`
|
|
prefixFirstSeenV6SQL = `SELECT first_seen FROM prefixes_v6 WHERE prefix = ?`
|
|
)
|
|
|
|
// GetPrefixFirstSeen returns when the prefix tables first saw prefix, or the
|
|
// zero time when they have no record of it.
|
|
func (d *Database) GetPrefixFirstSeen(
|
|
ctx context.Context, prefix netip.Prefix,
|
|
) (time.Time, error) {
|
|
query := prefixFirstSeenV6SQL
|
|
if prefix.Addr().Is4() {
|
|
query = prefixFirstSeenV4SQL
|
|
}
|
|
|
|
var firstSeen time.Time
|
|
|
|
err := d.db.QueryRowContext(ctx, query, prefix.Masked().String()).
|
|
Scan(&firstSeen)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return time.Time{}, nil
|
|
}
|
|
|
|
if err != nil {
|
|
return time.Time{}, fmt.Errorf("failed to query prefix: %w", err)
|
|
}
|
|
|
|
return firstSeen, nil
|
|
}
|
|
|
|
// ASPeer represents a peering relationship with another AS including handle,
|
|
// description, and timestamps.
|
|
//
|
|
//nolint:tagliatelle // field names are the existing JSON API
|
|
type ASPeer struct {
|
|
ASN int `json:"asn"`
|
|
Handle string `json:"handle"`
|
|
Description string `json:"description"`
|
|
FirstSeen time.Time `json:"first_seen"`
|
|
LastSeen time.Time `json:"last_seen"`
|
|
}
|
|
|
|
// GetASPeers returns all ASes that peer with the given AS
|
|
func (d *Database) GetASPeers(asn int) ([]ASPeer, error) {
|
|
return d.GetASPeersContext(context.Background(), asn)
|
|
}
|
|
|
|
// GetASPeersContext returns all ASes that peer with the given AS with context
|
|
// support
|
|
func (d *Database) GetASPeersContext(
|
|
ctx context.Context, asn int,
|
|
) ([]ASPeer, error) {
|
|
query := `
|
|
SELECT
|
|
CASE
|
|
WHEN p.as_a = ? THEN p.as_b
|
|
ELSE p.as_a
|
|
END as peer_asn,
|
|
COALESCE(a.handle, '') as handle,
|
|
COALESCE(a.description, '') as description,
|
|
p.first_seen,
|
|
p.last_seen
|
|
FROM peerings p
|
|
LEFT JOIN asns a ON a.asn = CASE
|
|
WHEN p.as_a = ? THEN p.as_b
|
|
ELSE p.as_a
|
|
END
|
|
WHERE p.as_a = ? OR p.as_b = ?
|
|
ORDER BY peer_asn
|
|
`
|
|
|
|
rows, err := d.db.QueryContext(ctx, query, asn, asn, asn, asn)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to query AS peers: %w", err)
|
|
}
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
var peers []ASPeer
|
|
|
|
for rows.Next() {
|
|
var peer ASPeer
|
|
|
|
err := rows.Scan(&peer.ASN, &peer.Handle, &peer.Description,
|
|
&peer.FirstSeen, &peer.LastSeen)
|
|
if err != nil {
|
|
d.logger.Error("Failed to scan peer row", "error", err, "asn", asn)
|
|
|
|
continue
|
|
}
|
|
|
|
peers = append(peers, peer)
|
|
}
|
|
|
|
err = rows.Err()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error iterating AS peers: %w", err)
|
|
}
|
|
|
|
return peers, nil
|
|
}
|
|
|
|
// GetNextStaleASN returns a random ASN that needs WHOIS data refresh.
|
|
func (d *Database) GetNextStaleASN(
|
|
ctx context.Context, staleThreshold time.Duration,
|
|
) (int, error) {
|
|
cutoff := time.Now().Add(-staleThreshold)
|
|
|
|
// Select a random stale ASN using ORDER BY RANDOM()
|
|
query := `
|
|
SELECT asn FROM asns
|
|
WHERE whois_updated_at IS NULL
|
|
OR whois_updated_at < ?
|
|
ORDER BY RANDOM()
|
|
LIMIT 1
|
|
`
|
|
|
|
var asn int
|
|
|
|
err := d.db.QueryRowContext(ctx, query, cutoff).Scan(&asn)
|
|
if err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return 0, ErrNoStaleASN
|
|
}
|
|
|
|
return 0, fmt.Errorf("failed to get stale ASN: %w", err)
|
|
}
|
|
|
|
return asn, nil
|
|
}
|
|
|
|
// WHOISStats contains statistics about WHOIS data freshness.
|
|
//
|
|
//nolint:tagliatelle // field names are the existing JSON API
|
|
type WHOISStats struct {
|
|
TotalASNs int `json:"total_asns"`
|
|
StaleASNs int `json:"stale_asns"`
|
|
FreshASNs int `json:"fresh_asns"`
|
|
NeverFetched int `json:"never_fetched"`
|
|
}
|
|
|
|
// GetWHOISStats returns statistics about WHOIS data freshness.
|
|
func (d *Database) GetWHOISStats(
|
|
ctx context.Context, staleThreshold time.Duration,
|
|
) (*WHOISStats, error) {
|
|
cutoff := time.Now().Add(-staleThreshold)
|
|
|
|
query := `
|
|
SELECT
|
|
COUNT(*) as total,
|
|
COALESCE(SUM(CASE WHEN whois_updated_at IS NULL
|
|
THEN 1 ELSE 0 END), 0) as never_fetched,
|
|
COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL
|
|
AND whois_updated_at < ? THEN 1 ELSE 0 END), 0) as stale,
|
|
COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL
|
|
AND whois_updated_at >= ? THEN 1 ELSE 0 END), 0) as fresh
|
|
FROM asns
|
|
`
|
|
|
|
var stats WHOISStats
|
|
|
|
err := d.db.QueryRowContext(ctx, query, cutoff, cutoff).Scan(
|
|
&stats.TotalASNs,
|
|
&stats.NeverFetched,
|
|
&stats.StaleASNs,
|
|
&stats.FreshASNs,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get WHOIS stats: %w", err)
|
|
}
|
|
|
|
return &stats, nil
|
|
}
|
|
|
|
// UpdateASNWHOIS updates an ASN record with WHOIS data.
|
|
func (d *Database) UpdateASNWHOIS(
|
|
ctx context.Context, update *ASNWHOISUpdate,
|
|
) error {
|
|
d.lock("UpdateASNWHOIS")
|
|
defer d.unlock()
|
|
|
|
query := `
|
|
UPDATE asns SET
|
|
as_name = ?,
|
|
org_name = ?,
|
|
org_id = ?,
|
|
address = ?,
|
|
country_code = ?,
|
|
abuse_email = ?,
|
|
abuse_phone = ?,
|
|
tech_email = ?,
|
|
tech_phone = ?,
|
|
rir = ?,
|
|
rir_registration_date = ?,
|
|
rir_last_modified = ?,
|
|
whois_raw = ?,
|
|
whois_updated_at = ?
|
|
WHERE asn = ?
|
|
`
|
|
|
|
_, err := d.writer.ExecContext(ctx, query,
|
|
update.ASName,
|
|
update.OrgName,
|
|
update.OrgID,
|
|
update.Address,
|
|
update.CountryCode,
|
|
update.AbuseEmail,
|
|
update.AbusePhone,
|
|
update.TechEmail,
|
|
update.TechPhone,
|
|
update.RIR,
|
|
update.RIRRegDate,
|
|
update.RIRLastMod,
|
|
update.WHOISRaw,
|
|
time.Now(),
|
|
update.ASN,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to update ASN WHOIS: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Vacuum runs incremental vacuum to return up to 1000 unused pages to the
|
|
// filesystem. Like Analyze and Checkpoint, it holds the database lock, so it
|
|
// and the batch writes wait for each other instead of failing on SQLite's
|
|
// write lock, and returns ctx's error without running if ctx ended while it
|
|
// waited: the writing connection it uses does not check ctx itself.
|
|
func (d *Database) Vacuum(ctx context.Context) error {
|
|
// Free up to 1000 pages per call (~4MB with default 4KB page size)
|
|
// This keeps each vacuum operation quick
|
|
const pagesToFree = 1000
|
|
|
|
d.lock("Vacuum")
|
|
defer d.unlock()
|
|
|
|
err := ctx.Err()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to run incremental vacuum: %w", err)
|
|
}
|
|
|
|
rows, err := d.writer.QueryContext(ctx,
|
|
fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to run incremental vacuum: %w", err)
|
|
}
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
// SQLite frees one page each time the statement steps, and each step
|
|
// returns a row, so every row must be read for the pragma to free more
|
|
// than one page.
|
|
for rows.Next() {
|
|
}
|
|
|
|
err = rows.Err()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to run incremental vacuum: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Analyze updates the query planner statistics. It runs ANALYZE, through
|
|
// PRAGMA optimize, only on the tables that have an index without statistics or
|
|
// whose size has changed tenfold since they were last analyzed: even with
|
|
// analysisLimit, ANALYZE counts every entry of each index, so it takes longer
|
|
// as the tables grow. It holds the database lock.
|
|
func (d *Database) Analyze(ctx context.Context) error {
|
|
d.lock("Analyze")
|
|
defer d.unlock()
|
|
|
|
err := ctx.Err()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to analyze database: %w", err)
|
|
}
|
|
|
|
// analysis_limit is a setting of the connection, so it is set in the same
|
|
// call as PRAGMA optimize, which runs both statements on one connection.
|
|
_, err = d.writer.ExecContext(ctx, fmt.Sprintf(
|
|
"PRAGMA analysis_limit=%d; PRAGMA optimize=%#x",
|
|
analysisLimit, optimizeAllTables))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to analyze database: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Checkpoint copies the WAL into the database file. It holds the database
|
|
// lock, so batch writes wait while it runs.
|
|
//
|
|
// It runs a PASSIVE checkpoint, which never waits: it copies what no open read
|
|
// still needs, and leaves the file as large as it is. When the file is over
|
|
// sqliteWALSizeLimitBytes, Checkpoint runs one TRUNCATE checkpoint instead,
|
|
// which waits at most truncateBusyTimeoutMs for the open reads to end, then
|
|
// copies all of the WAL and empties the file, or gives up. It is logged with
|
|
// how long it took and whether it emptied the file.
|
|
//
|
|
// So a file over the limit is emptied by the first Checkpoint that finds no
|
|
// read open, at its start or at some moment of its wait. While reads follow
|
|
// one another with no pause between them, every TRUNCATE checkpoint can give
|
|
// up after its full wait, and the file can keep growing for as long as they
|
|
// do.
|
|
//
|
|
// It returns ErrCheckpointIncomplete when it leaves part of the WAL uncopied,
|
|
// or a file over the limit not emptied.
|
|
func (d *Database) Checkpoint(ctx context.Context) error {
|
|
d.lock("Checkpoint")
|
|
defer d.unlock()
|
|
|
|
err := ctx.Err()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to checkpoint WAL: %w", err)
|
|
}
|
|
|
|
wal, err := os.Stat(d.path + walFileSuffix)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read the WAL file's size: %w", err)
|
|
}
|
|
|
|
if wal.Size() > sqliteWALSizeLimitBytes {
|
|
return d.truncateWAL(ctx, wal.Size())
|
|
}
|
|
|
|
var busy, walPages, copiedPages int
|
|
|
|
err = d.writer.QueryRowContext(ctx, "PRAGMA wal_checkpoint(PASSIVE)").
|
|
Scan(&busy, &walPages, &copiedPages)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to checkpoint WAL: %w", err)
|
|
}
|
|
|
|
if busy != 0 {
|
|
return fmt.Errorf("%w: SQLite reported it busy", ErrCheckpointIncomplete)
|
|
}
|
|
|
|
if copiedPages < walPages {
|
|
return fmt.Errorf("%w: copied %d of %d WAL pages",
|
|
ErrCheckpointIncomplete, copiedPages, walPages)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Ping checks if the database connection is alive with a lightweight query.
|
|
func (d *Database) Ping(ctx context.Context) error {
|
|
var result int
|
|
|
|
err := d.db.QueryRowContext(ctx, "SELECT 1").Scan(&result)
|
|
if err != nil {
|
|
return fmt.Errorf("database ping failed: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// readOneAS returns the AS that query reads, and whether it read one.
|
|
func (d *Database) readOneAS(ctx context.Context, query string) (*ASN, bool, error) {
|
|
var (
|
|
asnInfo ASN
|
|
handle, description sql.NullString
|
|
)
|
|
|
|
err := d.db.QueryRowContext(ctx, query).Scan(&asnInfo.ASN, &handle,
|
|
&description, &asnInfo.FirstSeen, &asnInfo.LastSeen)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil, false, nil
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("failed to query AS by last seen: %w", err)
|
|
}
|
|
|
|
asnInfo.Handle = handle.String
|
|
asnInfo.Description = description.String
|
|
|
|
return &asnInfo, true, nil
|
|
}
|
|
|
|
// truncateWAL runs the TRUNCATE checkpoint of Checkpoint on a WAL file of
|
|
// walBytes, with the writing connection's busy timeout lowered to
|
|
// truncateBusyTimeoutMs for it, and logs how long it took. The caller holds the
|
|
// database lock.
|
|
func (d *Database) truncateWAL(ctx context.Context, walBytes int64) error {
|
|
_, err := d.writer.ExecContext(ctx,
|
|
fmt.Sprintf("PRAGMA busy_timeout=%d", truncateBusyTimeoutMs))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to lower the busy timeout to truncate WAL: %w",
|
|
err)
|
|
}
|
|
|
|
started := time.Now()
|
|
|
|
var busy, walPages, copiedPages int
|
|
|
|
checkpointErr := d.writer.QueryRowContext(ctx,
|
|
"PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walPages, &copiedPages)
|
|
took := time.Since(started)
|
|
|
|
// Restored even when ctx has ended, so that no batch write runs with the
|
|
// short timeout.
|
|
_, restoreErr := d.writer.ExecContext(context.WithoutCancel(ctx),
|
|
fmt.Sprintf("PRAGMA busy_timeout=%d", sqliteBusyTimeoutMs))
|
|
if checkpointErr != nil || restoreErr != nil {
|
|
return fmt.Errorf("failed to truncate WAL: %w",
|
|
errors.Join(checkpointErr, restoreErr))
|
|
}
|
|
|
|
if busy != 0 {
|
|
d.logger.Warn("WAL file over its size limit not truncated",
|
|
"wal_bytes", walBytes, "wal_pages", walPages,
|
|
"copied_pages", copiedPages, "duration", took)
|
|
|
|
return fmt.Errorf("%w: WAL file of %d bytes not truncated",
|
|
ErrCheckpointIncomplete, walBytes)
|
|
}
|
|
|
|
d.logger.Info("WAL file over its size limit truncated",
|
|
"wal_bytes", walBytes, "duration", took)
|
|
|
|
return nil
|
|
}
|
|
|
|
// The debug messages lock logs, with the operation's name, when it starts to
|
|
// wait for the database mutex and when it holds it.
|
|
const (
|
|
lockWaitMessage = "Acquiring database lock"
|
|
lockHeldMessage = "Database lock acquired"
|
|
)
|
|
|
|
// lock acquires the database mutex and logs debug information
|
|
func (d *Database) lock(operation string) {
|
|
// Get caller information
|
|
_, file, line, _ := runtime.Caller(1)
|
|
caller := fmt.Sprintf("%s:%d", filepath.Base(file), line)
|
|
|
|
d.logger.Debug(lockWaitMessage, "operation", operation, "caller", caller)
|
|
|
|
d.mu.Lock()
|
|
d.lockedAt = time.Now()
|
|
d.lockedBy = fmt.Sprintf("%s (%s)", operation, caller)
|
|
|
|
d.logger.Debug(lockHeldMessage, "operation", operation, "caller", caller)
|
|
}
|
|
|
|
// unlock releases the database mutex and logs debug information including
|
|
// hold duration
|
|
func (d *Database) unlock() {
|
|
holdDuration := time.Since(d.lockedAt)
|
|
lockedBy := d.lockedBy
|
|
|
|
d.lockedAt = time.Time{}
|
|
d.lockedBy = ""
|
|
d.mu.Unlock()
|
|
|
|
d.logger.Debug("Database lock released",
|
|
"held_by", lockedBy, "duration_ms", holdDuration.Milliseconds())
|
|
}
|
|
|
|
// beginTx starts a new transaction on the writing connection, with logging
|
|
func (d *Database) beginTx() (*loggingTx, error) {
|
|
tx, err := d.writer.BeginTx(context.Background(), nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &loggingTx{Tx: tx, logger: d.logger}, nil
|
|
}
|
|
|
|
// rollback rolls back tx unless it has already been committed, and logs a
|
|
// failure. Every write defers it right after beginning its transaction.
|
|
func (d *Database) rollback(tx *loggingTx) {
|
|
err := tx.Rollback()
|
|
if err != nil && !errors.Is(err, sql.ErrTxDone) {
|
|
d.logger.Error("Failed to rollback transaction", "error", err)
|
|
}
|
|
}
|