Bound shutdown hooks by their stop context (closes #102)
All checks were successful
check / check (push) Successful in 3m45s
All checks were successful
check / check (push) Successful in 3m45s
fx hands OnStop a context carrying the application's stop timeout, and the delivery engine, the retention reaper, and the archive sweeper all discarded it and called wg.Wait() bare. A worker wedged inside a delivery target that never returns, or a sweep blocked on a locked SQLite database, hung the process forever instead of letting it exit when the timeout expired. All three now wait through internal/lifecycle.WaitForShutdown, which selects the drained WaitGroup against the stop context and, on timeout, logs at error naming the component and returns an error rather than reporting a clean stop. Engine.stop also gains the cancel != nil guard its two mirrored components already had.
This commit is contained in:
57
internal/lifecycle/lifecycle.go
Normal file
57
internal/lifecycle/lifecycle.go
Normal file
@@ -0,0 +1,57 @@
|
||||
// Package lifecycle holds helpers shared by the components that
|
||||
// register fx start and stop hooks.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// WaitForShutdown waits for wg to drain, bounded by ctx.
|
||||
//
|
||||
// fx hands OnStop a context carrying the application's stop
|
||||
// timeout. A bare wg.Wait() discards that deadline, so a single
|
||||
// goroutine that never observes cancellation — a delivery target
|
||||
// that never returns, a SQLite operation blocked on a lock —
|
||||
// hangs the process forever instead of letting it exit when the
|
||||
// timeout expires, which is exactly when a clean shutdown matters
|
||||
// most.
|
||||
//
|
||||
// On timeout it logs at error naming component and returns an
|
||||
// error: the goroutines are still running, and reporting success
|
||||
// would hide an unclean shutdown from the operator. The waiting
|
||||
// goroutine outlives this call and exits when (if) wg drains; it
|
||||
// holds nothing but the channel it closes.
|
||||
func WaitForShutdown(
|
||||
ctx context.Context,
|
||||
log *slog.Logger,
|
||||
component string,
|
||||
wg *sync.WaitGroup,
|
||||
) error {
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
defer close(done)
|
||||
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
log.Error(
|
||||
"shutdown timed out, goroutines still running",
|
||||
"component", component,
|
||||
"error", ctx.Err(),
|
||||
)
|
||||
|
||||
return fmt.Errorf(
|
||||
"%s: shutdown timed out, "+
|
||||
"goroutines still running: %w",
|
||||
component, ctx.Err(),
|
||||
)
|
||||
}
|
||||
}
|
||||
62
internal/lifecycle/lifecycle_test.go
Normal file
62
internal/lifecycle/lifecycle_test.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package lifecycle_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
||||
)
|
||||
|
||||
// waitTimeout is the stop budget the timeout case gives a
|
||||
// goroutine that never returns. The test's own patience is the
|
||||
// go test deadline, so the only thing this value affects is how
|
||||
// long the case takes.
|
||||
const waitTimeout = 100 * time.Millisecond
|
||||
|
||||
func discardLogger() *slog.Logger {
|
||||
return slog.New(slog.DiscardHandler)
|
||||
}
|
||||
|
||||
func TestWaitForShutdown_DrainedGroup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
wg.Go(func() {})
|
||||
|
||||
require.NoError(
|
||||
t,
|
||||
lifecycle.WaitForShutdown(
|
||||
context.Background(), discardLogger(),
|
||||
"test component", &wg,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func TestWaitForShutdown_ContextExpires(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
release := make(chan struct{})
|
||||
|
||||
t.Cleanup(func() { close(release) })
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
wg.Go(func() { <-release })
|
||||
|
||||
ctx, cancel := context.WithTimeout(
|
||||
context.Background(), waitTimeout,
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
err := lifecycle.WaitForShutdown(
|
||||
ctx, discardLogger(), "test component", &wg,
|
||||
)
|
||||
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
require.ErrorContains(t, err, "test component")
|
||||
}
|
||||
Reference in New Issue
Block a user