package middleware_test import ( "context" "fmt" "sync" "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sneak.berlin/go/webhooker/internal/database" "sneak.berlin/go/webhooker/internal/middleware" ) // mib converts the Argon2id memory parameter, which is in KiB, to MB. const mib = 1024 const ( // guardInterval is the failure window these tests use. It is // long enough that nothing lapses mid-test on its own; tests // that need a lapse drive the clock instead. guardInterval = time.Minute // guardWait is the slot wait for tests that expect to get a // slot. Tests that expect to be refused set their own. guardWait = 2 * time.Second guardClient = "198.51.100.7" guardUser = "admin" // racePasses is how many times a both-cases-ready select race is // run. A pass can only go the wrong way once the zero-duration // timer has fired, so the per-pass detection probability is // somewhere below 1/2 rather than exactly it; the bound that // matters is that passes are independent, so a regression that // survives is exponentially unlikely in N. The test still waits // on nothing. racePasses = 1000 ) // newGuard builds a guard with production-shaped defaults and the // given key-set cap and verification concurrency. func newGuard(maxKeys, concurrency int) *middleware.LoginGuard { return middleware.NewLoginGuardForTest( middleware.LoginRateLimitConst, guardInterval, maxKeys, concurrency, middleware.PasswordVerifyMaxWaitersConst, guardWait, ) } // TestLoginGuard_ThrottlesRepeatedFailures is the brute-force half: // wrong passwords for one username from one client key still run out // of budget and are answered 429. func TestLoginGuard_ThrottlesRepeatedFailures(t *testing.T) { t.Parallel() g := newGuard(middleware.LoginFailureMaxKeysConst, 1) for i := range middleware.LoginRateLimitConst - 1 { assert.False( t, g.FailForTest(guardClient, guardUser), "failure %d is still inside the budget", i, ) } assert.True( t, g.FailForTest(guardClient, guardUser), "the last failure of the budget must throttle", ) assert.True( t, g.FailForTest(guardClient, guardUser), "failures past the budget must stay throttled", ) } // TestLoginGuard_SuccessForgivesFailures pins the forgiveness rule: // an operator who mistypes several times and then gets it right must // not be left throttled. func TestLoginGuard_SuccessForgivesFailures(t *testing.T) { t.Parallel() g := newGuard(middleware.LoginFailureMaxKeysConst, 1) for range middleware.LoginRateLimitConst { g.FailForTest(guardClient, guardUser) } g.SucceedForTest(guardClient, guardUser) assert.False( t, g.FailForTest(guardClient, guardUser), "a success must reset the counter, so the next mistake "+ "starts a fresh budget", ) } // TestLoginGuard_FailuresAreKeyedPerUsername proves the second half // of the keying: one username's spent budget does not throttle // another's from the same client. func TestLoginGuard_FailuresAreKeyedPerUsername(t *testing.T) { t.Parallel() g := newGuard(middleware.LoginFailureMaxKeysConst, 1) for range middleware.LoginRateLimitConst { g.FailForTest(guardClient, guardUser) } assert.True(t, g.FailForTest(guardClient, guardUser)) assert.False( t, g.FailForTest(guardClient, "someone-else"), "a different submitted username must have its own budget", ) } // TestLoginGuard_WindowLapses covers the interval: a counter that has // gone quiet for the whole window starts again from zero. func TestLoginGuard_WindowLapses(t *testing.T) { t.Parallel() g := newGuard(middleware.LoginFailureMaxKeysConst, 1) var now atomic.Int64 now.Store(time.Now().UnixNano()) g.SetNowForTest(func() time.Time { return time.Unix(0, now.Load()) }) for range middleware.LoginRateLimitConst { g.FailForTest(guardClient, guardUser) } assert.True(t, g.FailForTest(guardClient, guardUser)) now.Add(int64(guardInterval) + 1) assert.False( t, g.FailForTest(guardClient, guardUser), "a lapsed window must start a fresh budget", ) } // TestLoginGuard_UsernameKeySetIsBounded is the memory bound. The // submitted username is attacker-controlled, so an attacker rotating // usernames must not be able to grow the guard without limit: past // the cap, tracking falls back to a counter keyed on the client // address alone. func TestLoginGuard_UsernameKeySetIsBounded(t *testing.T) { t.Parallel() const ( maxKeys = 8 attempts = 500 ) g := newGuard(maxKeys, 1) for i := range attempts { g.FailForTest(guardClient, fmt.Sprintf("user-%d", i)) } byUser, byAddr := g.TrackedKeysForTest() assert.LessOrEqual( t, byUser, maxKeys, "the per-username key set must not grow past its cap", ) assert.LessOrEqual( t, byAddr, maxKeys, "the fallback key set must not grow past its cap either", ) assert.Positive( t, byAddr, "past the cap, failures must fall back to the address "+ "bucket rather than being dropped", ) assert.Less( t, byUser+byAddr, attempts, "memory must not grow with the number of distinct "+ "usernames submitted", ) } // TestLoginGuard_BeyondBothCapsStaysThrottled covers the hard stop. // When both key sets are full of live counters and the client is in // neither, there is nothing to count without unbounded growth, so the // failure is answered as throttled. That costs the operator nothing: // a correct password never reaches this path. func TestLoginGuard_BeyondBothCapsStaysThrottled(t *testing.T) { t.Parallel() const maxKeys = 4 g := newGuard(maxKeys, 1) // Fill the per-username set from one client, then fill the // address set from distinct clients. for i := range maxKeys { g.FailForTest(guardClient, fmt.Sprintf("user-%d", i)) } for i := range maxKeys { g.FailForTest(fmt.Sprintf("203.0.113.%d", i), "whoever") } assert.True( t, g.FailForTest("203.0.113.200", "brand-new"), "a client that fits in neither full key set must be "+ "answered as throttled rather than tracked", ) byUser, byAddr := g.TrackedKeysForTest() assert.LessOrEqual(t, byUser, maxKeys) assert.LessOrEqual(t, byAddr, maxKeys) } // TestLoginGuard_SemaphoreBoundsConcurrentVerifications is the memory // bound on the hashing itself. Verifying credentials before spending // limiter budget means an attacker can force one Argon2id hash per // request, and each allocates 64 MB; without this bound the fix for // an admin lockout would be a memory-exhaustion DoS instead. func TestLoginGuard_SemaphoreBoundsConcurrentVerifications( t *testing.T, ) { t.Parallel() const ( concurrency = 2 workers = 12 // rendezvousDeadlock is the deadlock guard described below. // It is orders of magnitude longer than any scheduling delay, // so it never decides the result, and well inside script/test's // 30s timeout, so a wedge fails on the assertion instead of // blowing the package timeout. rendezvousDeadlock = 5 * time.Second ) g := newGuard(middleware.LoginFailureMaxKeysConst, concurrency) var ( mu sync.Mutex inside int highest int wg sync.WaitGroup recorded sync.WaitGroup once sync.Once ) // Slot holders rendezvous instead of sleeping, and they hold until // every worker has been answered. A sleep only makes overlap // likely — on a host loaded enough to deschedule a goroutine for // longer than the sleep the workers serialise and the maximum // observed comes back as 1 — so the rendezvous is what makes the // overlap a fact rather than a race won. // // The barrier must not open at the concurrency-th holder, which // would fix the lower bound at the cost of the upper one this test // exists to enforce: holders would leave as soon as the count // reached concurrency, so a guard admitting extra requests would // let them arrive after the first holders had already left and // highest would report concurrency however many were really let // in. It opens instead once every worker's acquire has returned // and any slot it won has been counted, so under a broken guard // every admitted worker is inside simultaneously and highest is // the true maximum. Under a correct guard the refused workers // return within the guard's own wait, which decides nothing beyond // how long that takes. overlapped := make(chan struct{}) closeOverlapped := func() { once.Do(func() { close(overlapped) }) } // Deadlock guard, not a timing margin: no assertion depends on its // length, and the only way to reach it is a worker that never // returns from acquire at all. It is here so that such a wedge // fails legibly on the assertion below instead of hanging until // the package test timeout. abandon := time.AfterFunc(rendezvousDeadlock, closeOverlapped) defer abandon.Stop() recorded.Add(workers) go func() { recorded.Wait() closeOverlapped() }() for range workers { wg.Go(func() { release, ok := g.AcquireForTest(context.Background()) if !ok { recorded.Done() return } defer release() mu.Lock() inside++ if inside > highest { highest = inside } mu.Unlock() // Counted before signalling, so the barrier can never open // while an admitted worker is still on its way to being // counted. recorded.Done() <-overlapped mu.Lock() inside-- mu.Unlock() }) } wg.Wait() mu.Lock() defer mu.Unlock() assert.Equal( t, concurrency, highest, "no more than %d verifications may run at once", concurrency, ) } // TestLoginGuard_SaturatedSemaphoreRefusesRatherThanQueueing pins // what happens when every slot is taken for longer than the wait: the // request is refused, so the caller answers 503 without allocating // another 64 MB hash. // // Neither half of this rides on the wait being long enough. The // refusal holds the only slot across the whole of the second call, so // there is no wait it could get lucky with — the wait fixes only how // long the refusal takes, not whether it happens. The reuse after // release is settled by acquire's non-blocking preamble, which is // pinned separately by TestLoginGuard_FreeSlotBeatsAnExpiredWait. So // the wait below is sized to keep the test quick, not to win a race. func TestLoginGuard_SaturatedSemaphoreRefusesRatherThanQueueing( t *testing.T, ) { t.Parallel() g := middleware.NewLoginGuardForTest( middleware.LoginRateLimitConst, guardInterval, middleware.LoginFailureMaxKeysConst, 1, middleware.PasswordVerifyMaxWaitersConst, 10*time.Millisecond, ) release, ok := g.AcquireForTest(context.Background()) require.True(t, ok, "the first acquire must get the only slot") _, ok = g.AcquireForTest(context.Background()) assert.False( t, ok, "with the only slot held, a second request must be refused "+ "rather than wait indefinitely", ) release() release, ok = g.AcquireForTest(context.Background()) // require, not assert: acquire returns a nil release alongside a // false ok, so calling it after a non-fatal assertion turns one // failed test into a segfault that takes the whole package test // binary down. Every assertion whose value is dereferenced later // has to stop the test. require.True( t, ok, "the slot must be reusable once released", ) release() } // TestLoginGuard_FreeSlotBeatsAnExpiredWait is the determinism this // file used to lack. acquire selects over a slot send and a wait // timer, and Go chooses among ready cases uniformly at random, so a // call made after the timer had already fired was a coin flip: on a // loaded host the previous test's third acquire could be refused // with its slot standing free, and then dereference the nil release // it got back. // // The wait here is already elapsed on arrival, which is the worst // case that scheduling can produce, so a free slot must still be // granted every time. Without acquire's non-blocking preamble each // pass is an independent coin flip and the loop fails within a few // passes; with it the property holds by construction and no wall // clock is involved. func TestLoginGuard_FreeSlotBeatsAnExpiredWait(t *testing.T) { t.Parallel() g := middleware.NewLoginGuardForTest( middleware.LoginRateLimitConst, guardInterval, middleware.LoginFailureMaxKeysConst, 1, middleware.PasswordVerifyMaxWaitersConst, 0, ) for pass := range racePasses { release, ok := g.AcquireForTest(context.Background()) require.Truef( t, ok, "pass %d was refused a slot that was free; an expired "+ "wait must never beat an available slot", pass, ) release() } } // TestLoginGuard_AcquireHonoursCancellation proves a client that // disconnects while queued frees its place immediately instead of // holding it for the full wait. func TestLoginGuard_AcquireHonoursCancellation(t *testing.T) { t.Parallel() g := newGuard(middleware.LoginFailureMaxKeysConst, 1) release, ok := g.AcquireForTest(context.Background()) require.True(t, ok) defer release() ctx, cancel := context.WithCancel(context.Background()) cancel() _, ok = g.AcquireForTest(ctx) assert.False( t, ok, "a cancelled request must not wait for a slot", ) } // TestPasswordVerifyConcurrency_MatchesMemoryBudget pins the // concurrency constant to the arithmetic behind it: the number of // slots is the hashing budget divided by what one Argon2id hash // actually costs. // // The per-hash figure is read out of the shipped password // parameters rather than copied here. A guard that asserts a literal // against a literal cannot see the thing it guards: raising // argon2Memory would leave it green while the real ceiling doubled. func TestPasswordVerifyConcurrency_MatchesMemoryBudget(t *testing.T) { t.Parallel() // Memory is the real argon2Memory, in KiB. perHashMB := int(database.DefaultPasswordConfig().Memory) / mib require.Positive( t, perHashMB, "the Argon2id memory parameter must be readable in MB", ) // The memory this service commits to password hashing. const budgetMB = 128 assert.Equal( t, middleware.PasswordVerifyConcurrencyConst, budgetMB/perHashMB, "the verification concurrency must be the %d MB hashing "+ "budget divided by the %d MB one Argon2id hash costs; "+ "if the Argon2id parameters changed, the slot count "+ "must change with them", budgetMB, perHashMB, ) } // TestLoginGuard_ShedsPastTheQueueCap pins the memory bound on // waiting, as distinct from the bound on hashing. A waiter arrives // with its form already parsed, and the retained parse plus its // header block cost several MB — far more than maxFormBodySize // suggests, since that caps only the raw body read — so an unbounded // queue would hold that much per waiting request for the whole wait; // past the cap the guard must refuse instantly rather than grow. func TestLoginGuard_ShedsPastTheQueueCap(t *testing.T) { t.Parallel() const ( maxWaiters = 2 // Long enough that a queued waiter never times out on its // own, so anything the test observes leaving the queue left // because it was shed. neverElapses = time.Minute // The probe carries its own deadline, so a guard that queues // the probe instead of shedding it fails here rather than // hanging until the package test timeout. // // This is a patience budget, not a margin to be won. A shed // returns in microseconds and a probe that queued instead // would not return for neverElapses, so the two are a whole // minute apart and any budget between them separates them. It // is set far above any scheduling stall a loaded host can // produce, because the previous 200 ms — and the 100 ms // elapsed-time assertion it fed — bounded the latency of a // goroutine hand-off, which is a false red waiting to happen // on the machine this suite runs on. What actually proves the // probe was not queued is the queue depth asserted below. probePatience = 5 * time.Second ) g := middleware.NewLoginGuardForTest( middleware.LoginRateLimitConst, guardInterval, middleware.LoginFailureMaxKeysConst, 1, maxWaiters, neverElapses, ) // Occupy the only slot, so everything after this queues. release, ok := g.AcquireForTest(context.Background()) require.True(t, ok) defer release() defer fillQueue(t, g, maxWaiters)() granted, answered := probeQueueCap(g, probePatience) require.True( t, answered, "a request arriving past the queue cap is still waiting to "+ "be queued; it must have been shed", ) assert.False( t, granted, "a request arriving past the queue cap must be shed", ) assert.Equal( t, maxWaiters, g.QueuedWaitersForTest(), "a shed request must not have grown the queue", ) } // fillQueue starts n waiters on g and returns once all of them are // queued for a slot. The returned function releases them and waits // for them to exit. func fillQueue( t *testing.T, g *middleware.LoginGuard, n int, ) func() { t.Helper() ctx, cancel := context.WithCancel(context.Background()) var wg sync.WaitGroup for range n { wg.Go(func() { done, got := g.AcquireForTest(ctx) if got { done() } }) } // Patience budget, not a margin: the waiters park in microseconds // and nothing releases them, so the only way to exhaust this is a // guard that never queues. One second is the same order as the // scheduling stalls this suite has to survive, so it is not one. require.Eventually( t, func() bool { return g.QueuedWaitersForTest() == n }, 5*time.Second, time.Millisecond, "the waiters must reach the queue before the cap is tested", ) return func() { cancel() wg.Wait() } } // probeQueueCap acquires from another goroutine. It reports, in // order, whether the call was granted a slot and whether it was // answered at all within wait; a call that never returned reports // false for both. // // It runs off the test goroutine deliberately. Joining a full queue // is not cancellable by context — refusing to join is the property // under test — so a guard that fails this would otherwise hang the // package until the test timeout instead of failing here. // // It reports no elapsed time. Timing a goroutine hand-off measures // the host, not the guard, and the caller distinguishes shedding from // queueing by the queue depth instead. func probeQueueCap( g *middleware.LoginGuard, wait time.Duration, ) (bool, bool) { probed := make(chan bool, 1) go func() { release, ok := g.AcquireForTest(context.Background()) if ok { release() } probed <- ok }() select { case result := <-probed: return result, true case <-time.After(wait): return false, false } }