Bound the WHOIS fetcher's success and error time lists (closes #118) #143

Merged
clawbot merged 1 commits from issue-118-fetcher-stats into next 2026-10-20 03:38:52 +02:00
2 changed files with 50 additions and 13 deletions
+18 -13
View File
@@ -53,7 +53,8 @@ type ASNFetcher struct {
currentInterval time.Duration currentInterval time.Duration
consecutiveFails int consecutiveFails int
// hourly stats tracking // hourly stats tracking; now is the clock, time.Now except in tests
now func() time.Time
statsMu sync.Mutex statsMu sync.Mutex
successTimes []time.Time successTimes []time.Time
errorTimes []time.Time errorTimes []time.Time
@@ -69,6 +70,7 @@ func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
stopCh: make(chan struct{}), stopCh: make(chan struct{}),
done: make(chan struct{}), done: make(chan struct{}),
currentInterval: baseInterval, currentInterval: baseInterval,
now: time.Now,
successTimes: make([]time.Time, 0), successTimes: make([]time.Time, 0),
errorTimes: make([]time.Time, 0), errorTimes: make([]time.Time, 0),
} }
@@ -121,7 +123,7 @@ func (f *ASNFetcher) GetStats() server.ASNFetcherStats {
f.intervalMu.Unlock() f.intervalMu.Unlock()
// Prune old entries and count // Prune old entries and count
cutoff := time.Now().Add(-statsWindow) cutoff := f.now().Add(-statsWindow)
f.successTimes = pruneOldTimes(f.successTimes, cutoff) f.successTimes = pruneOldTimes(f.successTimes, cutoff)
f.errorTimes = pruneOldTimes(f.errorTimes, cutoff) f.errorTimes = pruneOldTimes(f.errorTimes, cutoff)
@@ -133,16 +135,15 @@ func (f *ASNFetcher) GetStats() server.ASNFetcherStats {
} }
} }
// pruneOldTimes removes times older than cutoff and returns the pruned slice. // pruneOldTimes drops the times at or before cutoff from the front of times,
// which holds times in the order they were recorded, and returns the rest. It
// copies nothing, so it is cheap enough to run on every append.
func pruneOldTimes(times []time.Time, cutoff time.Time) []time.Time { func pruneOldTimes(times []time.Time, cutoff time.Time) []time.Time {
result := make([]time.Time, 0, len(times)) for len(times) > 0 && !times[0].After(cutoff) {
for _, t := range times { times = times[1:]
if t.After(cutoff) {
result = append(result, t)
}
} }
return result return times
} }
// getInterval returns the current fetch interval. // getInterval returns the current fetch interval.
@@ -170,9 +171,11 @@ func (f *ASNFetcher) recordSuccess() {
} }
f.intervalMu.Unlock() f.intervalMu.Unlock()
// Record success time for stats // Record success time for stats, dropping those older than statsWindow
f.statsMu.Lock() f.statsMu.Lock()
f.successTimes = append(f.successTimes, time.Now()) now := f.now()
f.successTimes = pruneOldTimes(f.successTimes, now.Add(-statsWindow))
f.successTimes = append(f.successTimes, now)
f.statsMu.Unlock() f.statsMu.Unlock()
} }
@@ -194,9 +197,11 @@ func (f *ASNFetcher) recordFailure() {
} }
f.intervalMu.Unlock() f.intervalMu.Unlock()
// Record error time for stats // Record error time for stats, dropping those older than statsWindow
f.statsMu.Lock() f.statsMu.Lock()
f.errorTimes = append(f.errorTimes, time.Now()) now := f.now()
f.errorTimes = pruneOldTimes(f.errorTimes, now.Add(-statsWindow))
f.errorTimes = append(f.errorTimes, now)
f.statsMu.Unlock() f.statsMu.Unlock()
} }
@@ -203,3 +203,35 @@ func TestASNFetcherStopReturnsByItsDeadline(t *testing.T) {
t.Fatal("the fetcher goroutine did not return after the lookup ended") t.Fatal("the fetcher goroutine did not return after the lookup ended")
} }
} }
// TestRecordKeepsOnlyTheStatsWindow records a success and a failure every
// second, by a clock the test sets, far longer than statsWindow, and never
// asks for the stats. The fetcher must keep only the times within the last
// statsWindow.
func TestRecordKeepsOnlyTheStatsWindow(t *testing.T) {
t.Parallel()
const (
records = 100_000
step = time.Second
want = int(statsWindow / step)
)
clock := time.Now()
f := NewASNFetcher(newMockStore(), logger.New().Logger)
f.now = func() time.Time { return clock }
for range records {
clock = clock.Add(step)
f.recordSuccess()
f.recordFailure()
}
if len(f.successTimes) != want || len(f.errorTimes) != want {
t.Errorf("%d success and %d error times kept, want %d of each, "+
"those of the last %s", len(f.successTimes), len(f.errorTimes),
want, statsWindow)
}
}