Store a new row's earliest timestamp as first_seen (closes #112)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
Each handler took the latest time per key in a batch and inserted it as a new row's first_seen, so a key updated twice in one batch, seconds apart, was stored first seen at the later time, and the IP lookup could answer with a prefix first_seen after the route's last_seen. The prefix handler now keeps the earliest time of all of the batch's updates for each prefix. The AS, peer and peering handlers keep the earliest and the latest time per key in database.SeenTimes, and GetOrCreateASNBatch, UpdatePeerBatch and RecordPeering store a new row first seen at the earliest and last seen at the latest. A stored row still has only its last_seen set, to the latest. Model: opus-5-5
This commit was merged in pull request #137.
This commit is contained in:
@@ -148,7 +148,9 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
|
||||
func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
|
||||
t.Helper()
|
||||
|
||||
err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts})
|
||||
seen := SeenTimes{First: ts, Last: ts}
|
||||
|
||||
err := db.GetOrCreateASNBatch(map[int]SeenTimes{64500: seen, 64501: seen})
|
||||
if err != nil {
|
||||
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
||||
}
|
||||
@@ -164,14 +166,14 @@ func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
|
||||
err = db.UpdatePeerBatch(map[string]PeerUpdate{
|
||||
testV4PeerIP: {
|
||||
PeerIP: testV4PeerIP, PeerASN: 64500,
|
||||
MessageType: "UPDATE", Timestamp: ts,
|
||||
MessageType: "UPDATE", Seen: seen,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UpdatePeerBatch: %v", err)
|
||||
}
|
||||
|
||||
err = db.RecordPeering(64500, 64501, ts)
|
||||
err = db.RecordPeering(64500, 64501, seen)
|
||||
if err != nil {
|
||||
t.Fatalf("RecordPeering: %v", err)
|
||||
}
|
||||
@@ -192,7 +194,10 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
|
||||
|
||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
|
||||
err = db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts})
|
||||
seen := SeenTimes{First: ts, Last: ts}
|
||||
|
||||
err = db.GetOrCreateASNBatch(
|
||||
map[int]SeenTimes{64500: seen, 64501: seen, 64502: seen})
|
||||
if err != nil {
|
||||
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
||||
}
|
||||
|
||||
@@ -903,13 +903,14 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// newASN returns a new ASN record first and last seen at timestamp, with the
|
||||
// handle and description from the AS info lookup when it knows the number.
|
||||
func newASN(number int, timestamp time.Time) ASN {
|
||||
// newASN returns a new ASN record first seen at seen.First and last seen at
|
||||
// seen.Last, with the handle and description from the AS info lookup when it
|
||||
// knows the number.
|
||||
func newASN(number int, seen SeenTimes) ASN {
|
||||
asn := ASN{
|
||||
ASN: number,
|
||||
FirstSeen: timestamp,
|
||||
LastSeen: timestamp,
|
||||
FirstSeen: seen.First,
|
||||
LastSeen: seen.Last,
|
||||
}
|
||||
|
||||
// Look up ASN info
|
||||
@@ -921,12 +922,12 @@ func newASN(number int, timestamp time.Time) ASN {
|
||||
return asn
|
||||
}
|
||||
|
||||
// upsertASN updates the last_seen time of ASN number with the given
|
||||
// upsertASN sets the last_seen time of ASN number to seen.Last with the given
|
||||
// statements, inserting the ASN when it is new. It reports whether it inserted
|
||||
// it.
|
||||
func upsertASN(
|
||||
ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt,
|
||||
number int, timestamp time.Time,
|
||||
number int, seen SeenTimes,
|
||||
) (bool, error) {
|
||||
var (
|
||||
asn ASN
|
||||
@@ -937,7 +938,7 @@ func upsertASN(
|
||||
&description, &asn.FirstSeen, &asn.LastSeen)
|
||||
if err == nil {
|
||||
// ASN exists, update last_seen
|
||||
_, err = updateStmt.ExecContext(ctx, timestamp, number)
|
||||
_, err = updateStmt.ExecContext(ctx, seen.Last, number)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to update ASN %d: %w", number, err)
|
||||
}
|
||||
@@ -950,7 +951,7 @@ func upsertASN(
|
||||
}
|
||||
|
||||
// ASN doesn't exist, create it
|
||||
asn = newASN(number, timestamp)
|
||||
asn = newASN(number, seen)
|
||||
|
||||
_, err = insertStmt.ExecContext(ctx, asn.ASN, asn.Handle,
|
||||
asn.Description, asn.FirstSeen, asn.LastSeen)
|
||||
@@ -962,7 +963,7 @@ func upsertASN(
|
||||
}
|
||||
|
||||
// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction
|
||||
func (d *Database) GetOrCreateASNBatch(asns map[int]time.Time) error {
|
||||
func (d *Database) GetOrCreateASNBatch(asns map[int]SeenTimes) error {
|
||||
if len(asns) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -1004,9 +1005,9 @@ func (d *Database) GetOrCreateASNBatch(asns map[int]time.Time) error {
|
||||
|
||||
var newASNs int
|
||||
|
||||
for number, timestamp := range asns {
|
||||
for number, seen := range asns {
|
||||
inserted, err := upsertASN(ctx, selectStmt, updateStmt, insertStmt,
|
||||
number, timestamp)
|
||||
number, seen)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1075,7 +1076,7 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
||||
}
|
||||
|
||||
// ASN doesn't exist, create it with ASN info lookup
|
||||
asn = newASN(number, timestamp)
|
||||
asn = newASN(number, SeenTimes{First: timestamp, Last: timestamp})
|
||||
|
||||
_, err = tx.Exec("INSERT INTO asns "+
|
||||
"(asn, handle, description, first_seen, last_seen) "+
|
||||
@@ -1182,7 +1183,7 @@ func (d *Database) RecordAnnouncement(announcement *Announcement) error {
|
||||
}
|
||||
|
||||
// RecordPeering records a peering relationship between two ASNs.
|
||||
func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error {
|
||||
func (d *Database) RecordPeering(asA, asB int, seen SeenTimes) error {
|
||||
// Validate ASNs
|
||||
if asA <= 0 || asB <= 0 {
|
||||
return fmt.Errorf("%w: asA=%d, asB=%d", errInvalidASN, asA, asB)
|
||||
@@ -1218,12 +1219,12 @@ func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error {
|
||||
if exists {
|
||||
_, err = tx.Exec(
|
||||
"UPDATE peerings SET last_seen = ? WHERE as_a = ? AND as_b = ?",
|
||||
timestamp, asA, asB)
|
||||
seen.Last, asA, asB)
|
||||
} else {
|
||||
_, err = tx.Exec(`
|
||||
INSERT INTO peerings (id, as_a, as_b, first_seen, last_seen)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
generateUUID().String(), asA, asB, timestamp, timestamp)
|
||||
generateUUID().String(), asA, asB, seen.First, seen.Last)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -1301,11 +1302,11 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error {
|
||||
|
||||
if exists {
|
||||
_, err = updateStmt.ExecContext(ctx, update.PeerASN,
|
||||
update.Timestamp, update.MessageType, update.PeerIP)
|
||||
update.Seen.Last, update.MessageType, update.PeerIP)
|
||||
} else {
|
||||
_, err = insertStmt.ExecContext(ctx,
|
||||
generateUUID().String(), update.PeerIP, update.PeerASN,
|
||||
update.Timestamp, update.Timestamp, update.MessageType)
|
||||
update.Seen.First, update.Seen.Last, update.MessageType)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -615,10 +615,12 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) {
|
||||
})
|
||||
|
||||
ts := time.Now().UTC()
|
||||
seen := SeenTimes{First: ts, Last: ts}
|
||||
|
||||
for i := range contentionIterations {
|
||||
asns := map[int]time.Time{
|
||||
i % contendedASNCount: ts,
|
||||
(i % contendedASNCount) + asnSecondBand: ts,
|
||||
asns := map[int]SeenTimes{
|
||||
i % contendedASNCount: seen,
|
||||
(i % contendedASNCount) + asnSecondBand: seen,
|
||||
}
|
||||
|
||||
err := db.GetOrCreateASNBatch(asns)
|
||||
@@ -747,7 +749,8 @@ func TestBatchWriteWaitsForAnalyze(t *testing.T) {
|
||||
|
||||
db, messages := newDatabaseWithLockMessages(t)
|
||||
ctx := context.Background()
|
||||
asns := map[int]time.Time{lockTestASN: time.Now().UTC()}
|
||||
now := time.Now().UTC()
|
||||
asns := map[int]SeenTimes{lockTestASN: {First: now, Last: now}}
|
||||
|
||||
// A stored AS gives ANALYZE a table to analyze and statistics to write.
|
||||
err := db.GetOrCreateASNBatch(asns)
|
||||
@@ -1663,6 +1666,48 @@ func TestUpdatePrefixesBatchKeepsFirstSeen(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdatePeerBatchStoresTheEarliestAndLatestTimesOfANewPeer writes a new
|
||||
// peer whose updates span 5 seconds. Its row is first seen at the earliest time
|
||||
// and last seen at the latest.
|
||||
func TestUpdatePeerBatchStoresTheEarliestAndLatestTimesOfANewPeer(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
first := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
last := first.Add(5 * time.Second)
|
||||
|
||||
err = db.UpdatePeerBatch(map[string]PeerUpdate{
|
||||
testPeerIP: {
|
||||
PeerIP: testPeerIP, PeerASN: 64500, MessageType: "UPDATE",
|
||||
Seen: SeenTimes{First: first, Last: last},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UpdatePeerBatch: %v", err)
|
||||
}
|
||||
|
||||
var firstSeen, lastSeen time.Time
|
||||
|
||||
err = db.db.QueryRowContext(t.Context(),
|
||||
"SELECT first_seen, last_seen FROM bgp_peers WHERE peer_ip = ?",
|
||||
testPeerIP).Scan(&firstSeen, &lastSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("reading peer %s: %v", testPeerIP, err)
|
||||
}
|
||||
|
||||
if !firstSeen.Equal(first) || !lastSeen.Equal(last) {
|
||||
t.Errorf("peer %s: first seen %s, last seen %s; want %s, %s",
|
||||
testPeerIP, firstSeen, lastSeen, first, last)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpsertLiveRouteBatchWritesInKeyOrder writes new routes of two peers out
|
||||
// of key order, the last one a second announcement of the first, and checks
|
||||
// that the rows were inserted in key order, peer then prefix (each new row's id
|
||||
|
||||
@@ -34,7 +34,7 @@ type Stats struct {
|
||||
type Store interface {
|
||||
// ASN operations
|
||||
GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
||||
GetOrCreateASNBatch(asns map[int]time.Time) error
|
||||
GetOrCreateASNBatch(asns map[int]SeenTimes) error
|
||||
|
||||
// Prefix operations
|
||||
GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error)
|
||||
@@ -44,7 +44,7 @@ type Store interface {
|
||||
RecordAnnouncement(announcement *Announcement) error
|
||||
|
||||
// Peering operations
|
||||
RecordPeering(asA, asB int, timestamp time.Time) error
|
||||
RecordPeering(asA, asB int, seen SeenTimes) error
|
||||
|
||||
// Statistics
|
||||
GetStats() (Stats, error)
|
||||
|
||||
@@ -122,12 +122,35 @@ type LiveRouteDeletion struct {
|
||||
IPVersion int
|
||||
}
|
||||
|
||||
// PeerUpdate represents parameters for updating a peer
|
||||
// SeenTimes is the earliest and the latest time one batch of updates saw an AS,
|
||||
// a peer or a peering. A row the batch adds is stored first seen at First and
|
||||
// last seen at Last; a row already stored keeps its first_seen and has its
|
||||
// last_seen set to Last.
|
||||
type SeenTimes struct {
|
||||
First time.Time
|
||||
Last time.Time
|
||||
}
|
||||
|
||||
// Add makes t the earliest time of s when it is earlier than First, and the
|
||||
// latest when it is later than Last. The zero SeenTimes holds no time yet, so
|
||||
// the first Add sets both.
|
||||
func (s *SeenTimes) Add(t time.Time) {
|
||||
if s.First.IsZero() || t.Before(s.First) {
|
||||
s.First = t
|
||||
}
|
||||
|
||||
if s.Last.IsZero() || t.After(s.Last) {
|
||||
s.Last = t
|
||||
}
|
||||
}
|
||||
|
||||
// PeerUpdate represents parameters for updating a peer. PeerASN and
|
||||
// MessageType are those of the peer's latest update.
|
||||
type PeerUpdate struct {
|
||||
PeerIP string
|
||||
PeerASN int
|
||||
MessageType string
|
||||
Timestamp time.Time
|
||||
Seen SeenTimes
|
||||
}
|
||||
|
||||
// ASNWHOISUpdate contains WHOIS data for updating an ASN record.
|
||||
|
||||
Reference in New Issue
Block a user