webhooker/internal/database/retention_lifecycle_test.go

package database_test

import (
	"context"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.uber.org/fx"
	"gorm.io/gorm"
	"sneak.berlin/go/webhooker/internal/database"
)

const (
	// reaperTestInterval is the sweep interval a lifecycle test
	// runs the reaper at, so a loop that survives startup produces
	// an observable sweep quickly.
	reaperTestInterval = 10 * time.Millisecond

	// reaperStopTimeout bounds how long a lifecycle test waits for
	// the reaper's OnStop hook to return before declaring the
	// shutdown hung.
	reaperStopTimeout = 10 * time.Second

	// reaperTestRetentionDays is the retention policy the lifecycle
	// tests give their webhook.
	reaperTestRetentionDays = 30
)

// recordingLifecycle is a minimal fx.Lifecycle that records the
// hooks a component registers, so a test can invoke the real
// OnStart/OnStop functions with a context of its choosing.
type recordingLifecycle struct {
	hooks []fx.Hook
}

func (l *recordingLifecycle) Append(h fx.Hook) {
	l.hooks = append(l.hooks, h)
}

// startReaperViaHook drives the genuine fx hooks the application
// registers for the reaper, handing OnStart a context that is
// already done. It returns the recorded lifecycle so the caller
// can drive OnStop too.
func startReaperViaHook(
	t *testing.T, r *database.RetentionReaper,
) *recordingLifecycle {
	t.Helper()

	lc := &recordingLifecycle{}
	r.ExportRegisterHooks(lc)
	require.Len(t, lc.hooks, 1)

	// fx hands OnStart a context carrying the application start
	// timeout, and cancels it when the start phase ends. An
	// already-cancelled context is that same defect taken to its
	// limit, and unlike a plain context.Background() it actually
	// distinguishes a correctly rooted loop from a broken one.
	hookCtx, cancel := context.WithCancel(context.Background())
	cancel()

	require.NoError(t, lc.hooks[0].OnStart(hookCtx))

	return lc
}

// eventGone reports whether an event row has been removed. It
// takes no *testing.T because it is polled from an
// assert.Eventually condition, which runs off the test goroutine
// where testify assertions must not be used.
func eventGone(db *gorm.DB, eventID string) bool {
	var n int64

	err := db.Unscoped().Model(&database.Event{}).
		Where("id = ?", eventID).Count(&n).Error
	if err != nil {
		return false
	}

	return n == 0
}

// seedExpiredWebhook creates a webhook with a finite retention
// policy plus one long-expired event chain, and returns the
// webhook's database and the chain's event ID.
func seedExpiredWebhook(
	t *testing.T, env *retentionTestEnv,
) (*gorm.DB, string) {
	t.Helper()

	webhookID := createWebhook(
		t, env.mainDB.DB(), reaperTestRetentionDays,
	)

	db, err := env.mgr.GetDB(webhookID)
	require.NoError(t, err)

	chain := seedEventChain(
		t, db, webhookID,
		time.Now().Add(-365*24*time.Hour),
	)

	return db, chain.eventID
}

// TestRetentionReaper_LoopOutlivesStartHookContext is the
// regression test for a reaper that never reaped. fx calls
// OnStart with a context carrying the application's start timeout
// (15s by default) and cancels it when the start phase ends, so a
// sweep loop rooted in it is dead three quarters of an hour
// before its first tick under the default one-hour interval, and
// per-webhook event databases grow without bound exactly as they
// did before retention existed.
//
// Driving OnStart with an already-cancelled context is that
// defect taken to its limit: a loop that inherits the hook
// context never ticks once, while a correctly rooted loop keeps
// sweeping for as long as the process lives.
func TestRetentionReaper_LoopOutlivesStartHookContext(
	t *testing.T,
) {
	t.Parallel()

	env := setupRetentionTest(t)

	db, eventID := seedExpiredWebhook(t, env)

	env.reaper.ExportSetInterval(reaperTestInterval)

	lc := startReaperViaHook(t, env.reaper)
	t.Cleanup(func() {
		_ = lc.hooks[0].OnStop(context.Background())
	})

	assert.Eventually(
		t,
		func() bool { return eventGone(db, eventID) },
		5*time.Second,
		reaperTestInterval,
		"the sweep loop must keep running after the start "+
			"hook's context is done; it reaped nothing, so it "+
			"inherited the hook context and died",
	)
}

// TestRetentionReaper_StopHookStopsLoop proves the fix did not
// trade a startup bug for a shutdown hang: now that the sweep
// loop no longer observes the start hook's cancellation, OnStop
// is the only thing that can stop it, and it must both return
// promptly and actually leave the loop stopped.
func TestRetentionReaper_StopHookStopsLoop(t *testing.T) {
	t.Parallel()

	env := setupRetentionTest(t)

	db, eventID := seedExpiredWebhook(t, env)

	env.reaper.ExportSetInterval(reaperTestInterval)

	lc := startReaperViaHook(t, env.reaper)

	// Let the loop prove it is running before stopping it, so a
	// fast OnStop cannot pass by stopping something already dead.
	require.Eventually(
		t,
		func() bool { return eventGone(db, eventID) },
		5*time.Second,
		reaperTestInterval,
	)

	var stopErr error

	stopped := make(chan struct{})

	go func() {
		defer close(stopped)

		// stop blocks on the loop's WaitGroup, so returning at all
		// proves the goroutine observed the cancellation.
		stopErr = lc.hooks[0].OnStop(context.Background())
	}()

	select {
	case <-stopped:
	case <-time.After(reaperStopTimeout):
		t.Fatal(
			"OnStop did not return: the retention reaper's " +
				"WaitGroup is still waiting on a loop that never " +
				"observed cancellation",
		)
	}

	require.NoError(t, stopErr)

	// With the loop gone, a newly expired chain must survive.
	survivor := seedEventChain(
		t, db, "stopped-webhook",
		time.Now().Add(-365*24*time.Hour),
	)

	time.Sleep(20 * reaperTestInterval)

	assert.False(
		t,
		eventGone(db, survivor.eventID),
		"a stopped reaper must not sweep anything",
	)
}