Bound the WHOIS fetcher's success and error time lists (closes #118) #143
@@ -53,7 +53,8 @@ type ASNFetcher struct {
|
|||||||
currentInterval time.Duration
|
currentInterval time.Duration
|
||||||
consecutiveFails int
|
consecutiveFails int
|
||||||
|
|
||||||
// hourly stats tracking
|
// hourly stats tracking; now is the clock, time.Now except in tests
|
||||||
|
now func() time.Time
|
||||||
statsMu sync.Mutex
|
statsMu sync.Mutex
|
||||||
successTimes []time.Time
|
successTimes []time.Time
|
||||||
errorTimes []time.Time
|
errorTimes []time.Time
|
||||||
@@ -69,6 +70,7 @@ func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
|
|||||||
stopCh: make(chan struct{}),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
done: make(chan struct{}),
|
||||||
currentInterval: baseInterval,
|
currentInterval: baseInterval,
|
||||||
|
now: time.Now,
|
||||||
successTimes: make([]time.Time, 0),
|
successTimes: make([]time.Time, 0),
|
||||||
errorTimes: make([]time.Time, 0),
|
errorTimes: make([]time.Time, 0),
|
||||||
}
|
}
|
||||||
@@ -121,7 +123,7 @@ func (f *ASNFetcher) GetStats() server.ASNFetcherStats {
|
|||||||
f.intervalMu.Unlock()
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
// Prune old entries and count
|
// Prune old entries and count
|
||||||
cutoff := time.Now().Add(-statsWindow)
|
cutoff := f.now().Add(-statsWindow)
|
||||||
f.successTimes = pruneOldTimes(f.successTimes, cutoff)
|
f.successTimes = pruneOldTimes(f.successTimes, cutoff)
|
||||||
f.errorTimes = pruneOldTimes(f.errorTimes, cutoff)
|
f.errorTimes = pruneOldTimes(f.errorTimes, cutoff)
|
||||||
|
|
||||||
@@ -133,16 +135,15 @@ func (f *ASNFetcher) GetStats() server.ASNFetcherStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// pruneOldTimes removes times older than cutoff and returns the pruned slice.
|
// pruneOldTimes drops the times at or before cutoff from the front of times,
|
||||||
|
// which holds times in the order they were recorded, and returns the rest. It
|
||||||
|
// copies nothing, so it is cheap enough to run on every append.
|
||||||
func pruneOldTimes(times []time.Time, cutoff time.Time) []time.Time {
|
func pruneOldTimes(times []time.Time, cutoff time.Time) []time.Time {
|
||||||
result := make([]time.Time, 0, len(times))
|
for len(times) > 0 && !times[0].After(cutoff) {
|
||||||
for _, t := range times {
|
times = times[1:]
|
||||||
if t.After(cutoff) {
|
|
||||||
result = append(result, t)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return result
|
return times
|
||||||
}
|
}
|
||||||
|
|
||||||
// getInterval returns the current fetch interval.
|
// getInterval returns the current fetch interval.
|
||||||
@@ -170,9 +171,11 @@ func (f *ASNFetcher) recordSuccess() {
|
|||||||
}
|
}
|
||||||
f.intervalMu.Unlock()
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
// Record success time for stats
|
// Record success time for stats, dropping those older than statsWindow
|
||||||
f.statsMu.Lock()
|
f.statsMu.Lock()
|
||||||
f.successTimes = append(f.successTimes, time.Now())
|
now := f.now()
|
||||||
|
f.successTimes = pruneOldTimes(f.successTimes, now.Add(-statsWindow))
|
||||||
|
f.successTimes = append(f.successTimes, now)
|
||||||
f.statsMu.Unlock()
|
f.statsMu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -194,9 +197,11 @@ func (f *ASNFetcher) recordFailure() {
|
|||||||
}
|
}
|
||||||
f.intervalMu.Unlock()
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
// Record error time for stats
|
// Record error time for stats, dropping those older than statsWindow
|
||||||
f.statsMu.Lock()
|
f.statsMu.Lock()
|
||||||
f.errorTimes = append(f.errorTimes, time.Now())
|
now := f.now()
|
||||||
|
f.errorTimes = pruneOldTimes(f.errorTimes, now.Add(-statsWindow))
|
||||||
|
f.errorTimes = append(f.errorTimes, now)
|
||||||
f.statsMu.Unlock()
|
f.statsMu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -203,3 +203,35 @@ func TestASNFetcherStopReturnsByItsDeadline(t *testing.T) {
|
|||||||
t.Fatal("the fetcher goroutine did not return after the lookup ended")
|
t.Fatal("the fetcher goroutine did not return after the lookup ended")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRecordKeepsOnlyTheStatsWindow records a success and a failure every
|
||||||
|
// second, by a clock the test sets, far longer than statsWindow, and never
|
||||||
|
// asks for the stats. The fetcher must keep only the times within the last
|
||||||
|
// statsWindow.
|
||||||
|
func TestRecordKeepsOnlyTheStatsWindow(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
records = 100_000
|
||||||
|
step = time.Second
|
||||||
|
want = int(statsWindow / step)
|
||||||
|
)
|
||||||
|
|
||||||
|
clock := time.Now()
|
||||||
|
|
||||||
|
f := NewASNFetcher(newMockStore(), logger.New().Logger)
|
||||||
|
f.now = func() time.Time { return clock }
|
||||||
|
|
||||||
|
for range records {
|
||||||
|
clock = clock.Add(step)
|
||||||
|
|
||||||
|
f.recordSuccess()
|
||||||
|
f.recordFailure()
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(f.successTimes) != want || len(f.errorTimes) != want {
|
||||||
|
t.Errorf("%d success and %d error times kept, want %d of each, "+
|
||||||
|
"those of the last %s", len(f.successTimes), len(f.errorTimes),
|
||||||
|
want, statsWindow)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user