webhooker/internal/middleware/loginguard_test.go

package middleware_test

import (
	"context"
	"fmt"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"sneak.berlin/go/webhooker/internal/database"
	"sneak.berlin/go/webhooker/internal/middleware"
)

// mib converts the Argon2id memory parameter, which is in KiB, to MB.
const mib = 1024

const (
	// guardInterval is the failure window these tests use. It is
	// long enough that nothing lapses mid-test on its own; tests
	// that need a lapse drive the clock instead.
	guardInterval = time.Minute

	// guardWait is the slot wait for tests that expect to get a
	// slot. Tests that expect to be refused set their own.
	guardWait = 2 * time.Second

	guardClient = "198.51.100.7"
	guardUser   = "admin"

	// racePasses is how many times a both-cases-ready select race is
	// run. A pass can only go the wrong way once the zero-duration
	// timer has fired, so the per-pass detection probability is
	// somewhere below 1/2 rather than exactly it; the bound that
	// matters is that passes are independent, so a regression that
	// survives is exponentially unlikely in N. The test still waits
	// on nothing.
	racePasses = 1000
)

// newGuard builds a guard with production-shaped defaults and the
// given key-set cap and verification concurrency.
func newGuard(maxKeys, concurrency int) *middleware.LoginGuard {
	return middleware.NewLoginGuardForTest(
		middleware.LoginRateLimitConst,
		guardInterval,
		maxKeys,
		concurrency,
		middleware.PasswordVerifyMaxWaitersConst,
		guardWait,
	)
}

// TestLoginGuard_ThrottlesRepeatedFailures is the brute-force half:
// wrong passwords for one username from one client key still run out
// of budget and are answered 429.
func TestLoginGuard_ThrottlesRepeatedFailures(t *testing.T) {
	t.Parallel()

	g := newGuard(middleware.LoginFailureMaxKeysConst, 1)

	for i := range middleware.LoginRateLimitConst - 1 {
		assert.False(
			t, g.FailForTest(guardClient, guardUser),
			"failure %d is still inside the budget", i,
		)
	}

	assert.True(
		t, g.FailForTest(guardClient, guardUser),
		"the last failure of the budget must throttle",
	)
	assert.True(
		t, g.FailForTest(guardClient, guardUser),
		"failures past the budget must stay throttled",
	)
}

// TestLoginGuard_SuccessForgivesFailures pins the forgiveness rule:
// an operator who mistypes several times and then gets it right must
// not be left throttled.
func TestLoginGuard_SuccessForgivesFailures(t *testing.T) {
	t.Parallel()

	g := newGuard(middleware.LoginFailureMaxKeysConst, 1)

	for range middleware.LoginRateLimitConst {
		g.FailForTest(guardClient, guardUser)
	}

	g.SucceedForTest(guardClient, guardUser)

	assert.False(
		t, g.FailForTest(guardClient, guardUser),
		"a success must reset the counter, so the next mistake "+
			"starts a fresh budget",
	)
}

// TestLoginGuard_FailuresAreKeyedPerUsername proves the second half
// of the keying: one username's spent budget does not throttle
// another's from the same client.
func TestLoginGuard_FailuresAreKeyedPerUsername(t *testing.T) {
	t.Parallel()

	g := newGuard(middleware.LoginFailureMaxKeysConst, 1)

	for range middleware.LoginRateLimitConst {
		g.FailForTest(guardClient, guardUser)
	}

	assert.True(t, g.FailForTest(guardClient, guardUser))
	assert.False(
		t, g.FailForTest(guardClient, "someone-else"),
		"a different submitted username must have its own budget",
	)
}

// TestLoginGuard_WindowLapses covers the interval: a counter that has
// gone quiet for the whole window starts again from zero.
func TestLoginGuard_WindowLapses(t *testing.T) {
	t.Parallel()

	g := newGuard(middleware.LoginFailureMaxKeysConst, 1)

	var now atomic.Int64

	now.Store(time.Now().UnixNano())
	g.SetNowForTest(func() time.Time {
		return time.Unix(0, now.Load())
	})

	for range middleware.LoginRateLimitConst {
		g.FailForTest(guardClient, guardUser)
	}

	assert.True(t, g.FailForTest(guardClient, guardUser))

	now.Add(int64(guardInterval) + 1)

	assert.False(
		t, g.FailForTest(guardClient, guardUser),
		"a lapsed window must start a fresh budget",
	)
}

// TestLoginGuard_UsernameKeySetIsBounded is the memory bound. The
// submitted username is attacker-controlled, so an attacker rotating
// usernames must not be able to grow the guard without limit: past
// the cap, tracking falls back to a counter keyed on the client
// address alone.
func TestLoginGuard_UsernameKeySetIsBounded(t *testing.T) {
	t.Parallel()

	const (
		maxKeys  = 8
		attempts = 500
	)

	g := newGuard(maxKeys, 1)

	for i := range attempts {
		g.FailForTest(guardClient, fmt.Sprintf("user-%d", i))
	}

	byUser, byAddr := g.TrackedKeysForTest()

	assert.LessOrEqual(
		t, byUser, maxKeys,
		"the per-username key set must not grow past its cap",
	)
	assert.LessOrEqual(
		t, byAddr, maxKeys,
		"the fallback key set must not grow past its cap either",
	)
	assert.Positive(
		t, byAddr,
		"past the cap, failures must fall back to the address "+
			"bucket rather than being dropped",
	)
	assert.Less(
		t, byUser+byAddr, attempts,
		"memory must not grow with the number of distinct "+
			"usernames submitted",
	)
}

// TestLoginGuard_BeyondBothCapsStaysThrottled covers the hard stop.
// When both key sets are full of live counters and the client is in
// neither, there is nothing to count without unbounded growth, so the
// failure is answered as throttled. That costs the operator nothing:
// a correct password never reaches this path.
func TestLoginGuard_BeyondBothCapsStaysThrottled(t *testing.T) {
	t.Parallel()

	const maxKeys = 4

	g := newGuard(maxKeys, 1)

	// Fill the per-username set from one client, then fill the
	// address set from distinct clients.
	for i := range maxKeys {
		g.FailForTest(guardClient, fmt.Sprintf("user-%d", i))
	}

	for i := range maxKeys {
		g.FailForTest(fmt.Sprintf("203.0.113.%d", i), "whoever")
	}

	assert.True(
		t, g.FailForTest("203.0.113.200", "brand-new"),
		"a client that fits in neither full key set must be "+
			"answered as throttled rather than tracked",
	)

	byUser, byAddr := g.TrackedKeysForTest()
	assert.LessOrEqual(t, byUser, maxKeys)
	assert.LessOrEqual(t, byAddr, maxKeys)
}

// TestLoginGuard_SemaphoreBoundsConcurrentVerifications is the memory
// bound on the hashing itself. Verifying credentials before spending
// limiter budget means an attacker can force one Argon2id hash per
// request, and each allocates 64 MB; without this bound the fix for
// an admin lockout would be a memory-exhaustion DoS instead.
func TestLoginGuard_SemaphoreBoundsConcurrentVerifications(
	t *testing.T,
) {
	t.Parallel()

	const (
		concurrency = 2
		workers     = 12

		// rendezvousDeadlock is the deadlock guard described below.
		// It is orders of magnitude longer than any scheduling delay,
		// so it never decides the result, and well inside script/test's
		// 30s timeout, so a wedge fails on the assertion instead of
		// blowing the package timeout.
		rendezvousDeadlock = 5 * time.Second
	)

	g := newGuard(middleware.LoginFailureMaxKeysConst, concurrency)

	var (
		mu       sync.Mutex
		inside   int
		highest  int
		wg       sync.WaitGroup
		recorded sync.WaitGroup
		once     sync.Once
	)

	// Slot holders rendezvous instead of sleeping, and they hold until
	// every worker has been answered. A sleep only makes overlap
	// likely — on a host loaded enough to deschedule a goroutine for
	// longer than the sleep the workers serialise and the maximum
	// observed comes back as 1 — so the rendezvous is what makes the
	// overlap a fact rather than a race won.
	//
	// The barrier must not open at the concurrency-th holder, which
	// would fix the lower bound at the cost of the upper one this test
	// exists to enforce: holders would leave as soon as the count
	// reached concurrency, so a guard admitting extra requests would
	// let them arrive after the first holders had already left and
	// highest would report concurrency however many were really let
	// in. It opens instead once every worker's acquire has returned
	// and any slot it won has been counted, so under a broken guard
	// every admitted worker is inside simultaneously and highest is
	// the true maximum. Under a correct guard the refused workers
	// return within the guard's own wait, which decides nothing beyond
	// how long that takes.
	overlapped := make(chan struct{})
	closeOverlapped := func() {
		once.Do(func() { close(overlapped) })
	}

	// Deadlock guard, not a timing margin: no assertion depends on its
	// length, and the only way to reach it is a worker that never
	// returns from acquire at all. It is here so that such a wedge
	// fails legibly on the assertion below instead of hanging until
	// the package test timeout.
	abandon := time.AfterFunc(rendezvousDeadlock, closeOverlapped)
	defer abandon.Stop()

	recorded.Add(workers)

	go func() {
		recorded.Wait()
		closeOverlapped()
	}()

	for range workers {
		wg.Go(func() {
			release, ok := g.AcquireForTest(context.Background())
			if !ok {
				recorded.Done()

				return
			}

			defer release()

			mu.Lock()

			inside++
			if inside > highest {
				highest = inside
			}

			mu.Unlock()

			// Counted before signalling, so the barrier can never open
			// while an admitted worker is still on its way to being
			// counted.
			recorded.Done()

			<-overlapped

			mu.Lock()
			inside--
			mu.Unlock()
		})
	}

	wg.Wait()

	mu.Lock()
	defer mu.Unlock()

	assert.Equal(
		t, concurrency, highest,
		"no more than %d verifications may run at once", concurrency,
	)
}

// TestLoginGuard_SaturatedSemaphoreRefusesRatherThanQueueing pins
// what happens when every slot is taken for longer than the wait: the
// request is refused, so the caller answers 503 without allocating
// another 64 MB hash.
//
// Neither half of this rides on the wait being long enough. The
// refusal holds the only slot across the whole of the second call, so
// there is no wait it could get lucky with — the wait fixes only how
// long the refusal takes, not whether it happens. The reuse after
// release is settled by acquire's non-blocking preamble, which is
// pinned separately by TestLoginGuard_FreeSlotBeatsAnExpiredWait. So
// the wait below is sized to keep the test quick, not to win a race.
func TestLoginGuard_SaturatedSemaphoreRefusesRatherThanQueueing(
	t *testing.T,
) {
	t.Parallel()

	g := middleware.NewLoginGuardForTest(
		middleware.LoginRateLimitConst,
		guardInterval,
		middleware.LoginFailureMaxKeysConst,
		1,
		middleware.PasswordVerifyMaxWaitersConst,
		10*time.Millisecond,
	)

	release, ok := g.AcquireForTest(context.Background())
	require.True(t, ok, "the first acquire must get the only slot")

	_, ok = g.AcquireForTest(context.Background())
	assert.False(
		t, ok,
		"with the only slot held, a second request must be refused "+
			"rather than wait indefinitely",
	)

	release()

	release, ok = g.AcquireForTest(context.Background())

	// require, not assert: acquire returns a nil release alongside a
	// false ok, so calling it after a non-fatal assertion turns one
	// failed test into a segfault that takes the whole package test
	// binary down. Every assertion whose value is dereferenced later
	// has to stop the test.
	require.True(
		t, ok, "the slot must be reusable once released",
	)

	release()
}

// TestLoginGuard_FreeSlotBeatsAnExpiredWait is the determinism this
// file used to lack. acquire selects over a slot send and a wait
// timer, and Go chooses among ready cases uniformly at random, so a
// call made after the timer had already fired was a coin flip: on a
// loaded host the previous test's third acquire could be refused
// with its slot standing free, and then dereference the nil release
// it got back.
//
// The wait here is already elapsed on arrival, which is the worst
// case that scheduling can produce, so a free slot must still be
// granted every time. Without acquire's non-blocking preamble each
// pass is an independent coin flip and the loop fails within a few
// passes; with it the property holds by construction and no wall
// clock is involved.
func TestLoginGuard_FreeSlotBeatsAnExpiredWait(t *testing.T) {
	t.Parallel()

	g := middleware.NewLoginGuardForTest(
		middleware.LoginRateLimitConst,
		guardInterval,
		middleware.LoginFailureMaxKeysConst,
		1,
		middleware.PasswordVerifyMaxWaitersConst,
		0,
	)

	for pass := range racePasses {
		release, ok := g.AcquireForTest(context.Background())
		require.Truef(
			t, ok,
			"pass %d was refused a slot that was free; an expired "+
				"wait must never beat an available slot",
			pass,
		)

		release()
	}
}

// TestLoginGuard_AcquireHonoursCancellation proves a client that
// disconnects while queued frees its place immediately instead of
// holding it for the full wait.
func TestLoginGuard_AcquireHonoursCancellation(t *testing.T) {
	t.Parallel()

	g := newGuard(middleware.LoginFailureMaxKeysConst, 1)

	release, ok := g.AcquireForTest(context.Background())
	require.True(t, ok)

	defer release()

	ctx, cancel := context.WithCancel(context.Background())
	cancel()

	_, ok = g.AcquireForTest(ctx)
	assert.False(
		t, ok, "a cancelled request must not wait for a slot",
	)
}

// TestPasswordVerifyConcurrency_MatchesMemoryBudget pins the
// concurrency constant to the arithmetic behind it: the number of
// slots is the hashing budget divided by what one Argon2id hash
// actually costs.
//
// The per-hash figure is read out of the shipped password
// parameters rather than copied here. A guard that asserts a literal
// against a literal cannot see the thing it guards: raising
// argon2Memory would leave it green while the real ceiling doubled.
func TestPasswordVerifyConcurrency_MatchesMemoryBudget(t *testing.T) {
	t.Parallel()

	// Memory is the real argon2Memory, in KiB.
	perHashMB := int(database.DefaultPasswordConfig().Memory) / mib

	require.Positive(
		t, perHashMB,
		"the Argon2id memory parameter must be readable in MB",
	)

	// The memory this service commits to password hashing.
	const budgetMB = 128

	assert.Equal(
		t,
		middleware.PasswordVerifyConcurrencyConst,
		budgetMB/perHashMB,
		"the verification concurrency must be the %d MB hashing "+
			"budget divided by the %d MB one Argon2id hash costs; "+
			"if the Argon2id parameters changed, the slot count "+
			"must change with them",
		budgetMB, perHashMB,
	)
}

// TestLoginGuard_ShedsPastTheQueueCap pins the memory bound on
// waiting, as distinct from the bound on hashing. A waiter arrives
// with its form already parsed, and the retained parse plus its
// header block cost several MB — far more than maxFormBodySize
// suggests, since that caps only the raw body read — so an unbounded
// queue would hold that much per waiting request for the whole wait;
// past the cap the guard must refuse instantly rather than grow.
func TestLoginGuard_ShedsPastTheQueueCap(t *testing.T) {
	t.Parallel()

	const (
		maxWaiters = 2

		// Long enough that a queued waiter never times out on its
		// own, so anything the test observes leaving the queue left
		// because it was shed.
		neverElapses = time.Minute

		// The probe carries its own deadline, so a guard that queues
		// the probe instead of shedding it fails here rather than
		// hanging until the package test timeout.
		//
		// This is a patience budget, not a margin to be won. A shed
		// returns in microseconds and a probe that queued instead
		// would not return for neverElapses, so the two are a whole
		// minute apart and any budget between them separates them. It
		// is set far above any scheduling stall a loaded host can
		// produce, because the previous 200 ms — and the 100 ms
		// elapsed-time assertion it fed — bounded the latency of a
		// goroutine hand-off, which is a false red waiting to happen
		// on the machine this suite runs on. What actually proves the
		// probe was not queued is the queue depth asserted below.
		probePatience = 5 * time.Second
	)

	g := middleware.NewLoginGuardForTest(
		middleware.LoginRateLimitConst,
		guardInterval,
		middleware.LoginFailureMaxKeysConst,
		1,
		maxWaiters,
		neverElapses,
	)

	// Occupy the only slot, so everything after this queues.
	release, ok := g.AcquireForTest(context.Background())
	require.True(t, ok)

	defer release()
	defer fillQueue(t, g, maxWaiters)()

	granted, answered := probeQueueCap(g, probePatience)

	require.True(
		t, answered,
		"a request arriving past the queue cap is still waiting to "+
			"be queued; it must have been shed",
	)
	assert.False(
		t, granted,
		"a request arriving past the queue cap must be shed",
	)
	assert.Equal(
		t, maxWaiters, g.QueuedWaitersForTest(),
		"a shed request must not have grown the queue",
	)
}

// fillQueue starts n waiters on g and returns once all of them are
// queued for a slot. The returned function releases them and waits
// for them to exit.
func fillQueue(
	t *testing.T,
	g *middleware.LoginGuard,
	n int,
) func() {
	t.Helper()

	ctx, cancel := context.WithCancel(context.Background())

	var wg sync.WaitGroup

	for range n {
		wg.Go(func() {
			done, got := g.AcquireForTest(ctx)
			if got {
				done()
			}
		})
	}

	// Patience budget, not a margin: the waiters park in microseconds
	// and nothing releases them, so the only way to exhaust this is a
	// guard that never queues. One second is the same order as the
	// scheduling stalls this suite has to survive, so it is not one.
	require.Eventually(
		t,
		func() bool { return g.QueuedWaitersForTest() == n },
		5*time.Second, time.Millisecond,
		"the waiters must reach the queue before the cap is tested",
	)

	return func() {
		cancel()
		wg.Wait()
	}
}

// probeQueueCap acquires from another goroutine. It reports, in
// order, whether the call was granted a slot and whether it was
// answered at all within wait; a call that never returned reports
// false for both.
//
// It runs off the test goroutine deliberately. Joining a full queue
// is not cancellable by context — refusing to join is the property
// under test — so a guard that fails this would otherwise hang the
// package until the test timeout instead of failing here.
//
// It reports no elapsed time. Timing a goroutine hand-off measures
// the host, not the guard, and the caller distinguishes shedding from
// queueing by the queue depth instead.
func probeQueueCap(
	g *middleware.LoginGuard,
	wait time.Duration,
) (bool, bool) {
	probed := make(chan bool, 1)

	go func() {
		release, ok := g.AcquireForTest(context.Background())
		if ok {
			release()
		}

		probed <- ok
	}()

	select {
	case result := <-probed:
		return result, true
	case <-time.After(wait):
		return false, false
	}
}