All checks were successful
check / check (push) Successful in 3m3s
fx defaults to a 15s stop timeout and the Dockerfile sets no grace override, so Docker SIGKILLed at 10s and the bounded shutdown #130 built — including the log line that tells an operator a component is wedged — was unreachable in the image this repo produces. Sets fx.StopTimeout to 5s, and lowers the HTTP drain to 3s so a full-length drain no longer exhausts the whole sequence budget and skip every later hook, database close included. The Sentry flush, which runs in the same hook and honours no context, is clamped to the remaining stop budget less a 2s tail reserve, so a stalled flush drops Sentry events rather than the database close. Also fixes a latent coin flip in the shared stop-hook waiter, which reported "shutdown timed out" about half the time for a component that drained cleanly against an already-expired context. Independently reviewed three times. The final reviewer derived a stronger invariant than the implementation claims — the server hook's absolute end is bounded at stopTimeout minus the reserve regardless of drain length or of time consumed by preceding hooks — and confirmed the guard's 10ms sweep cannot step over the maximum, since both breakpoints land on its grid. Both Sentry probe arms, the docker stop demo and every mutation were reproduced independently. Known residual, filed separately: the HTTP drain itself is not clamped by the reserve, so slow preceding hooks can still jointly exhaust the budget. Demonstrated with a 2.2s sweeper delay.
243 lines
5.9 KiB
Go
243 lines
5.9 KiB
Go
// Package server wires up HTTP routes and manages the
|
|
// application lifecycle.
|
|
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"go.uber.org/fx"
|
|
"sneak.berlin/go/webhooker/internal/config"
|
|
"sneak.berlin/go/webhooker/internal/globals"
|
|
"sneak.berlin/go/webhooker/internal/handlers"
|
|
"sneak.berlin/go/webhooker/internal/logger"
|
|
"sneak.berlin/go/webhooker/internal/middleware"
|
|
|
|
"github.com/getsentry/sentry-go"
|
|
"github.com/go-chi/chi"
|
|
)
|
|
|
|
const (
|
|
// ShutdownTimeout is the maximum time to wait for the HTTP
|
|
// server to finish in-flight requests during shutdown.
|
|
//
|
|
// It must stay strictly below the fx stop timeout in
|
|
// cmd/webhooker, which bounds the whole stop sequence: a drain
|
|
// that used the entire sequence budget would leave nothing for
|
|
// the hooks that run after the server, including the database
|
|
// close. It is exported so that relationship can be tested.
|
|
ShutdownTimeout = 3 * time.Second
|
|
|
|
// TailHookReserve is the share of the fx stop budget this hook
|
|
// refuses to spend, leaving it for the hooks that run after the
|
|
// server: the delivery engine, the healthcheck, the webhook DB
|
|
// manager and the database close.
|
|
TailHookReserve = 2 * time.Second
|
|
|
|
// sentryFlushTimeout is the longest wait for Sentry to flush
|
|
// pending events during shutdown, before the remaining stop
|
|
// budget is taken into account.
|
|
sentryFlushTimeout = 2 * time.Second
|
|
|
|
// minSentryFlush is the shortest flush worth attempting. Below
|
|
// it the remaining budget goes to the tail hooks instead.
|
|
minSentryFlush = 250 * time.Millisecond
|
|
)
|
|
|
|
// SentryFlushBudget reports how long the Sentry flush may run when
|
|
// remaining is the time left on the fx stop context after the HTTP
|
|
// drain. sentry.Flush takes a bare duration and honours no context,
|
|
// so this clamp is the only thing keeping a stalled flush from
|
|
// spending the tail hooks' share of the budget on top of a
|
|
// full-length drain. TailHookReserve is held back, and anything
|
|
// under minSentryFlush is skipped rather than attempted uselessly.
|
|
func SentryFlushBudget(remaining time.Duration) time.Duration {
|
|
budget := min(remaining-TailHookReserve, sentryFlushTimeout)
|
|
if budget < minSentryFlush {
|
|
return 0
|
|
}
|
|
|
|
return budget
|
|
}
|
|
|
|
//nolint:revive // ServerParams is a standard fx naming convention.
|
|
type ServerParams struct {
|
|
fx.In
|
|
|
|
Logger *logger.Logger
|
|
Globals *globals.Globals
|
|
Config *config.Config
|
|
Middleware *middleware.Middleware
|
|
Handlers *handlers.Handlers
|
|
}
|
|
|
|
// Server is the main HTTP server that wires up routes and manages
|
|
// graceful shutdown.
|
|
type Server struct {
|
|
startupTime time.Time
|
|
exitCode int
|
|
sentryEnabled bool
|
|
log *slog.Logger
|
|
cancelFunc context.CancelFunc
|
|
httpServer *http.Server
|
|
router *chi.Mux
|
|
params ServerParams
|
|
mw *middleware.Middleware
|
|
h *handlers.Handlers
|
|
}
|
|
|
|
// New creates a Server that starts the HTTP listener on fx start
|
|
// and stops it gracefully.
|
|
func New(lc fx.Lifecycle, params ServerParams) (*Server, error) {
|
|
s := new(Server)
|
|
s.params = params
|
|
s.mw = params.Middleware
|
|
s.h = params.Handlers
|
|
s.log = params.Logger.Get()
|
|
|
|
lc.Append(fx.Hook{
|
|
OnStart: func(_ context.Context) error {
|
|
s.startupTime = time.Now()
|
|
go s.Run()
|
|
|
|
return nil
|
|
},
|
|
OnStop: func(ctx context.Context) error {
|
|
s.cleanShutdown(ctx)
|
|
|
|
return nil
|
|
},
|
|
})
|
|
|
|
return s, nil
|
|
}
|
|
|
|
// Run configures Sentry and starts serving HTTP requests.
|
|
func (s *Server) Run() {
|
|
s.configure()
|
|
|
|
// logging before sentry, because sentry logs
|
|
s.enableSentry()
|
|
|
|
s.serve()
|
|
}
|
|
|
|
// MaintenanceMode returns whether the server is in maintenance
|
|
// mode.
|
|
func (s *Server) MaintenanceMode() bool {
|
|
return s.params.Config.MaintenanceMode
|
|
}
|
|
|
|
func (s *Server) enableSentry() {
|
|
s.sentryEnabled = false
|
|
|
|
if s.params.Config.SentryDSN == "" {
|
|
return
|
|
}
|
|
|
|
err := sentry.Init(sentry.ClientOptions{
|
|
Dsn: s.params.Config.SentryDSN,
|
|
Release: fmt.Sprintf(
|
|
"%s-%s",
|
|
s.params.Globals.Appname,
|
|
s.params.Globals.Version,
|
|
),
|
|
})
|
|
if err != nil {
|
|
s.log.Error("sentry init failure", "error", err)
|
|
// Don't use fatal since we still want the service to run
|
|
return
|
|
}
|
|
|
|
s.log.Info("sentry error reporting activated")
|
|
s.sentryEnabled = true
|
|
}
|
|
|
|
func (s *Server) serve() int {
|
|
ctx, cancelFunc := context.WithCancel(context.Background())
|
|
s.cancelFunc = cancelFunc
|
|
|
|
// signal watcher
|
|
go func() {
|
|
c := make(chan os.Signal, 1)
|
|
|
|
signal.Ignore(syscall.SIGPIPE)
|
|
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
|
// block and wait for signal
|
|
sig := <-c
|
|
s.log.Info("signal received", "signal", sig.String())
|
|
|
|
if s.cancelFunc != nil {
|
|
// cancelling the main context will trigger a clean
|
|
// shutdown via the fx OnStop hook.
|
|
s.cancelFunc()
|
|
}
|
|
}()
|
|
|
|
go s.serveUntilShutdown()
|
|
|
|
<-ctx.Done()
|
|
// Shutdown is handled by the fx OnStop hook (cleanShutdown).
|
|
// Do not call cleanShutdown() here to avoid double invocation.
|
|
return s.exitCode
|
|
}
|
|
|
|
func (s *Server) cleanupForExit() {
|
|
s.log.Info("cleaning up")
|
|
}
|
|
|
|
func (s *Server) cleanShutdown(ctx context.Context) {
|
|
// initiate clean shutdown
|
|
s.exitCode = 0
|
|
|
|
ctxShutdown, shutdownCancel := context.WithTimeout(
|
|
ctx, ShutdownTimeout,
|
|
)
|
|
defer shutdownCancel()
|
|
|
|
err := s.httpServer.Shutdown(ctxShutdown)
|
|
if err != nil {
|
|
s.log.Error(
|
|
"server clean shutdown failed", "error", err,
|
|
)
|
|
}
|
|
|
|
s.cleanupForExit()
|
|
|
|
if s.sentryEnabled {
|
|
s.flushSentry(ctx)
|
|
}
|
|
}
|
|
|
|
// flushSentry drains Sentry's queue inside what is left of the fx
|
|
// stop budget. A context carrying no deadline — a caller outside the
|
|
// fx lifecycle — gets the full timeout.
|
|
func (s *Server) flushSentry(ctx context.Context) {
|
|
flush := sentryFlushTimeout
|
|
|
|
if deadline, ok := ctx.Deadline(); ok {
|
|
flush = SentryFlushBudget(time.Until(deadline))
|
|
}
|
|
|
|
if flush <= 0 {
|
|
s.log.Warn(
|
|
"skipping sentry flush, stop budget exhausted",
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
sentry.Flush(flush)
|
|
}
|
|
|
|
func (s *Server) configure() {
|
|
// identify ourselves in the logs
|
|
s.params.Logger.Identify()
|
|
}
|