Files
webhooker/internal/server/server.go
clawbot bfbf416393
All checks were successful
check / check (push) Successful in 6m20s
Exit non-zero when the HTTP listener fails (closes #200)
The fx OnStart hook returned as soon as the serving goroutine was
spawned, so a failed listen was discovered only inside that goroutine.
It logged the error and cancelled the server's own context, which
nothing outside the goroutine observes: fx reported RUNNING and the
process stayed alive with nothing bound. The service was down and
looked up, so systemd Restart=on-failure and Docker restart policies
never fired.

The Server now takes fx.Shutdowner and, on a listen failure, asks it to
stop the app with ListenFailureExitCode. Shutdown runs through fx's
normal stop sequence, so every OnStop hook — including the HTTP drain
and the database close — still executes; the clean-shutdown path is
untouched.

Server.exitCode goes with it. Its only writer set it to zero during
cleanShutdown and its only reader was serve()'s return value, which Run
discards; the process status is fx's to decide. Left in place it would
have been a genuine data race, since the new path makes cleanShutdown
and serve()'s return run concurrently under -race.

Verified by TestListenFailure_ShutsDownTheApp, which occupies a port,
starts the wired app on it, and requires a non-zero shutdown signal
within two seconds. Reverting the fix fails it with "listen failure
left the app running".
2026-08-20 04:13:53 +00:00

275 lines
7.3 KiB
Go

// Package server wires up HTTP routes and manages the
// application lifecycle.
package server
import (
"context"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"go.uber.org/fx"
"sneak.berlin/go/webhooker/internal/config"
"sneak.berlin/go/webhooker/internal/globals"
"sneak.berlin/go/webhooker/internal/handlers"
"sneak.berlin/go/webhooker/internal/logger"
"sneak.berlin/go/webhooker/internal/middleware"
"github.com/getsentry/sentry-go"
"github.com/go-chi/chi"
)
const (
// ShutdownTimeout is the maximum time to wait for the HTTP
// server to finish in-flight requests during shutdown.
//
// It must stay strictly below the fx stop timeout in
// cmd/webhooker, which bounds the whole stop sequence: a drain
// that used the entire sequence budget would leave nothing for
// the hooks that run after the server, including the database
// close. It is exported so that relationship can be tested.
ShutdownTimeout = 3 * time.Second
// TailHookReserve is the share of the fx stop budget this hook
// refuses to spend, leaving it for the hooks that run after the
// server: the delivery engine, the healthcheck, the webhook DB
// manager and the database close.
TailHookReserve = 2 * time.Second
// sentryFlushTimeout is the longest wait for Sentry to flush
// pending events during shutdown, before the remaining stop
// budget is taken into account.
sentryFlushTimeout = 2 * time.Second
// minSentryFlush is the shortest flush worth attempting. Below
// it the remaining budget goes to the tail hooks instead.
minSentryFlush = 250 * time.Millisecond
)
// ListenFailureExitCode is the status the process exits with when the
// HTTP listener cannot be established, or dies for a reason other
// than a requested shutdown. It must stay non-zero: systemd
// `Restart=on-failure` and Docker's restart policies key off it, and a
// zero exit would read as a deliberate stop.
const ListenFailureExitCode = 1
// SentryFlushBudget reports how long the Sentry flush may run when
// remaining is the time left on the fx stop context after the HTTP
// drain. sentry.Flush takes a bare duration and honours no context,
// so this clamp is the only thing keeping a stalled flush from
// spending the tail hooks' share of the budget on top of a
// full-length drain. TailHookReserve is held back, and anything
// under minSentryFlush is skipped rather than attempted uselessly.
func SentryFlushBudget(remaining time.Duration) time.Duration {
budget := min(remaining-TailHookReserve, sentryFlushTimeout)
if budget < minSentryFlush {
return 0
}
return budget
}
//nolint:revive // ServerParams is a standard fx naming convention.
type ServerParams struct {
fx.In
Logger *logger.Logger
Globals *globals.Globals
Config *config.Config
Middleware *middleware.Middleware
Handlers *handlers.Handlers
Shutdowner fx.Shutdowner
}
// Server is the main HTTP server that wires up routes and manages
// graceful shutdown.
type Server struct {
startupTime time.Time
sentryEnabled bool
log *slog.Logger
cancelFunc context.CancelFunc
httpServer *http.Server
router *chi.Mux
params ServerParams
mw *middleware.Middleware
h *handlers.Handlers
}
// New creates a Server that starts the HTTP listener on fx start
// and stops it gracefully.
func New(lc fx.Lifecycle, params ServerParams) (*Server, error) {
s := new(Server)
s.params = params
s.mw = params.Middleware
s.h = params.Handlers
s.log = params.Logger.Get()
lc.Append(fx.Hook{
OnStart: func(_ context.Context) error {
s.startupTime = time.Now()
go s.Run()
return nil
},
OnStop: func(ctx context.Context) error {
s.cleanShutdown(ctx)
return nil
},
})
return s, nil
}
// Run configures Sentry and starts serving HTTP requests.
func (s *Server) Run() {
s.configure()
// logging before sentry, because sentry logs
s.enableSentry()
s.serve()
}
// MaintenanceMode returns whether the server is in maintenance
// mode.
func (s *Server) MaintenanceMode() bool {
return s.params.Config.MaintenanceMode
}
func (s *Server) enableSentry() {
s.sentryEnabled = false
if s.params.Config.SentryDSN == "" {
return
}
err := sentry.Init(sentryClientOptions(
s.params.Config.SentryDSN,
fmt.Sprintf(
"%s-%s",
s.params.Globals.Appname,
s.params.Globals.Version,
),
))
if err != nil {
s.log.Error("sentry init failure", "error", err)
// Don't use fatal since we still want the service to run
return
}
s.log.Info("sentry error reporting activated")
s.sentryEnabled = true
}
// serve installs the signal watcher, starts the listener and blocks
// until the server's context is cancelled. The process exit status is
// fx's to decide — from a signal, or from the code
// shutdownOnListenFailure hands the Shutdowner — so this reports
// nothing back to its caller.
func (s *Server) serve() {
ctx, cancelFunc := context.WithCancel(context.Background())
s.cancelFunc = cancelFunc
// signal watcher
go func() {
c := make(chan os.Signal, 1)
signal.Ignore(syscall.SIGPIPE)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
// block and wait for signal
sig := <-c
s.log.Info("signal received", "signal", sig.String())
if s.cancelFunc != nil {
// cancelling the main context will trigger a clean
// shutdown via the fx OnStop hook.
s.cancelFunc()
}
}()
go s.serveUntilShutdown()
<-ctx.Done()
// Shutdown is handled by the fx OnStop hook (cleanShutdown).
// Do not call cleanShutdown() here to avoid double invocation.
}
// shutdownOnListenFailure ends the application after the HTTP
// listener failed. The fx OnStart hook returns as soon as the serving
// goroutine is spawned, so nothing downstream of it ever learns that
// the listen failed: fx reports RUNNING and the process sits alive
// with nothing bound, which is invisible to systemd and Docker
// restart policies. Asking the Shutdowner to stop the app with a
// non-zero code is what turns that into a visible failure.
//
// The context cancel that follows only unwinds serve()'s own wait.
// The shutdown itself runs through fx's normal stop sequence, so the
// clean-shutdown drain in cleanShutdown is reached unchanged.
func (s *Server) shutdownOnListenFailure() {
err := s.params.Shutdowner.Shutdown(
fx.ExitCode(ListenFailureExitCode),
)
if err != nil {
s.log.Error("shutdown request failed", "error", err)
}
if s.cancelFunc != nil {
s.cancelFunc()
}
}
func (s *Server) cleanupForExit() {
s.log.Info("cleaning up")
}
func (s *Server) cleanShutdown(ctx context.Context) {
ctxShutdown, shutdownCancel := context.WithTimeout(
ctx, ShutdownTimeout,
)
defer shutdownCancel()
err := s.httpServer.Shutdown(ctxShutdown)
if err != nil {
s.log.Error(
"server clean shutdown failed", "error", err,
)
}
s.cleanupForExit()
if s.sentryEnabled {
s.flushSentry(ctx)
}
}
// flushSentry drains Sentry's queue inside what is left of the fx
// stop budget. A context carrying no deadline — a caller outside the
// fx lifecycle — gets the full timeout.
func (s *Server) flushSentry(ctx context.Context) {
flush := sentryFlushTimeout
if deadline, ok := ctx.Deadline(); ok {
flush = SentryFlushBudget(time.Until(deadline))
}
if flush <= 0 {
s.log.Warn(
"skipping sentry flush, stop budget exhausted",
)
return
}
sentry.Flush(flush)
}
func (s *Server) configure() {
// identify ourselves in the logs
s.params.Logger.Identify()
}