Never move last seen back or first seen forward (closes #162)
check / check (push) Waiting to run
check / check (push) Waiting to run
Some RIS collectors send updates days old while they catch up on a backlog. Every update to a stored AS, peering or peer set last_seen to the update's time, even an older one, and never lowered first_seen. An AS page could then list a peer first seen after it was last seen. The update statements for asns, peerings and bgp_peers now keep the earlier of the stored and new first_seen and the later of the stored and new last_seen, with MIN and MAX in the SQL. The batch and the single-row writes share each statement. GetOrCreateASN returns the times the row now holds. Route times are unchanged. Model: opus-5-5
This commit is contained in:
@@ -20,6 +20,7 @@ import (
|
||||
|
||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||
"git.eeqj.de/sneak/routewatch/pkg/asinfo"
|
||||
"github.com/mattn/go-sqlite3"
|
||||
)
|
||||
@@ -1960,6 +1961,158 @@ func TestUpdatePeerBatchStoresTheEarliestAndLatestTimesOfANewPeer(
|
||||
}
|
||||
}
|
||||
|
||||
// backlogAge is how much older one write is than the other in the tests of
|
||||
// first_seen and last_seen, as when a collector catches up on a backlog.
|
||||
const backlogAge = 24 * time.Hour
|
||||
|
||||
// batchSpan is how much earlier a batch's First is than its Last in the tests
|
||||
// of first_seen and last_seen.
|
||||
const batchSpan = time.Hour
|
||||
|
||||
// TestWritesKeepEarliestFirstSeenAndLatestLastSeen writes a row of each of
|
||||
// asns, peerings and bgp_peers through each function that writes one, at a
|
||||
// newer time and at an older one. Each row takes the older write's First as
|
||||
// first_seen and the newer write's Last as last_seen. The test writes in both
|
||||
// orders, each in a database of its own. Newer then older is a collector
|
||||
// catching up on a backlog; older then newer is a feed that arrives in order.
|
||||
// Each function writes a row of its own. A batch write's First is batchSpan
|
||||
// before its Last, so the test catches the two swapped. A single-row write
|
||||
// takes one time.
|
||||
func TestWritesKeepEarliestFirstSeenAndLatestLastSeen(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
batchASN, singleASN = 64500, 64501
|
||||
peerASN = 64502
|
||||
asnQuery = "SELECT first_seen, last_seen FROM asns WHERE asn = ?"
|
||||
peeringQuery = "SELECT first_seen, last_seen FROM peerings " +
|
||||
"WHERE as_a = ? AND as_b = ?"
|
||||
peerQuery = "SELECT first_seen, last_seen FROM bgp_peers " +
|
||||
"WHERE peer_ip = ?"
|
||||
)
|
||||
|
||||
for name, olderFirst := range map[string]bool{
|
||||
"newer then older": false,
|
||||
"older then newer": true,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
||||
return db.GetOrCreateASNBatch(map[int]SeenTimes{batchASN: seen})
|
||||
}, batchSpan, asnQuery, batchASN)
|
||||
|
||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
||||
_, err := db.GetOrCreateASN(singleASN, seen.Last)
|
||||
|
||||
return err
|
||||
}, 0, asnQuery, singleASN)
|
||||
|
||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
||||
return db.RecordPeering(batchASN, singleASN, seen)
|
||||
}, batchSpan, peeringQuery, batchASN, singleASN)
|
||||
|
||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
||||
return db.UpdatePeerBatch(map[string]PeerUpdate{
|
||||
testV4PeerIP: {
|
||||
PeerIP: testV4PeerIP, PeerASN: peerASN,
|
||||
MessageType: ristypes.MessageTypeUpdate,
|
||||
Seen: seen,
|
||||
},
|
||||
})
|
||||
}, batchSpan, peerQuery, testV4PeerIP)
|
||||
|
||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
||||
return db.UpdatePeer(testPeerIP, peerASN,
|
||||
ristypes.MessageTypeUpdate, seen.Last)
|
||||
}, 0, peerQuery, testPeerIP)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// checkSeenTimes calls write twice, each time with First span before Last: at
|
||||
// newer times and at times backlogAge older. It writes the older times first
|
||||
// when olderFirst is true. It checks that query, given args, reads the older
|
||||
// write's First as first_seen and the newer write's Last as last_seen.
|
||||
func checkSeenTimes(
|
||||
t *testing.T, db *Database, olderFirst bool,
|
||||
write func(seen SeenTimes) error,
|
||||
span time.Duration, query string, args ...any,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
newerLast := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
newer := SeenTimes{First: newerLast.Add(-span), Last: newerLast}
|
||||
older := SeenTimes{
|
||||
First: newer.First.Add(-backlogAge),
|
||||
Last: newer.Last.Add(-backlogAge),
|
||||
}
|
||||
|
||||
writes := []SeenTimes{newer, older}
|
||||
if olderFirst {
|
||||
writes = []SeenTimes{older, newer}
|
||||
}
|
||||
|
||||
for _, seen := range writes {
|
||||
err := write(seen)
|
||||
if err != nil {
|
||||
t.Fatalf("writing first %s, last %s: %v", seen.First, seen.Last, err)
|
||||
}
|
||||
}
|
||||
|
||||
var firstSeen, lastSeen time.Time
|
||||
|
||||
err := db.db.QueryRowContext(t.Context(), query, args...).
|
||||
Scan(&firstSeen, &lastSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("reading the row: %v", err)
|
||||
}
|
||||
|
||||
if !firstSeen.Equal(older.First) || !lastSeen.Equal(newer.Last) {
|
||||
t.Errorf("first seen %s, last seen %s; want %s, %s",
|
||||
firstSeen, lastSeen, older.First, newer.Last)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetOrCreateASNReturnsStoredSeenTimes stores an AS at a newer time, then
|
||||
// at an older one. The second call returns the times the row now holds: the
|
||||
// older first_seen and the newer last_seen.
|
||||
func TestGetOrCreateASNReturnsStoredSeenTimes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
const asn = 64500
|
||||
|
||||
newer := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
older := newer.Add(-backlogAge)
|
||||
|
||||
_, err = db.GetOrCreateASN(asn, newer)
|
||||
if err != nil {
|
||||
t.Fatalf("GetOrCreateASN at %s: %v", newer, err)
|
||||
}
|
||||
|
||||
got, err := db.GetOrCreateASN(asn, older)
|
||||
if err != nil {
|
||||
t.Fatalf("GetOrCreateASN at %s: %v", older, err)
|
||||
}
|
||||
|
||||
if !got.FirstSeen.Equal(older) || !got.LastSeen.Equal(newer) {
|
||||
t.Errorf("first seen %s, last seen %s; want %s, %s",
|
||||
got.FirstSeen, got.LastSeen, older, newer)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpsertLiveRouteBatchWritesInKeyOrder writes new routes of two peers out
|
||||
// of key order, the last one a second announcement of the first, and checks
|
||||
// that the rows were inserted in key order, peer then prefix (each new row's id
|
||||
|
||||
Reference in New Issue
Block a user