check / check (push) Successful in 3m25s
MAINTENANCE_MODE did nothing but make the healthcheck JSON report maintenanceMode: true; no request was ever served differently, so an operator who set it expecting requests to be refused got nothing. It is removed from the configuration, the startup configuration log line, the healthcheck JSON, the README and the Settings page, together with the uncalled Server.MaintenanceMode method and the healthcheck's dependency on the configuration. A leftover value in an environment is ignored like any other unknown variable. Model: opus-5-5
309 lines
9.0 KiB
Go
309 lines
9.0 KiB
Go
// Package server wires up HTTP routes and manages the
|
|
// application lifecycle.
|
|
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"time"
|
|
|
|
"go.uber.org/fx"
|
|
"sneak.berlin/go/webhooker/internal/config"
|
|
"sneak.berlin/go/webhooker/internal/globals"
|
|
"sneak.berlin/go/webhooker/internal/handlers"
|
|
"sneak.berlin/go/webhooker/internal/logger"
|
|
"sneak.berlin/go/webhooker/internal/middleware"
|
|
|
|
"github.com/getsentry/sentry-go"
|
|
"github.com/go-chi/chi"
|
|
)
|
|
|
|
const (
|
|
// ShutdownTimeout is the maximum time to wait for the HTTP
|
|
// server to finish in-flight requests during shutdown.
|
|
//
|
|
// It must stay strictly below the fx stop timeout in
|
|
// cmd/webhooker, which bounds the whole stop sequence: a drain
|
|
// that used the entire sequence budget would leave nothing for
|
|
// the hooks that run after the server, including the database
|
|
// close. It is exported so that relationship can be tested.
|
|
ShutdownTimeout = 3 * time.Second
|
|
|
|
// TailHookReserve is the share of the fx stop budget this hook
|
|
// refuses to spend, leaving it for the hooks that run after the
|
|
// server: the delivery engine, the healthcheck, the webhook DB
|
|
// manager and the database close.
|
|
TailHookReserve = 2 * time.Second
|
|
|
|
// sentryFlushTimeout is the longest wait for Sentry to flush
|
|
// pending events during shutdown, before the remaining stop
|
|
// budget is taken into account.
|
|
sentryFlushTimeout = 2 * time.Second
|
|
|
|
// minSentryFlush is the shortest flush worth attempting. Below
|
|
// it the remaining budget goes to the tail hooks instead.
|
|
minSentryFlush = 250 * time.Millisecond
|
|
)
|
|
|
|
// StartupFailureExitCode is the status the process exits with when
|
|
// the serving goroutine gives up: the HTTP listener cannot be
|
|
// established or dies for a reason other than a requested shutdown, or
|
|
// error reporting is configured and cannot be started. It must stay
|
|
// non-zero: systemd `Restart=on-failure` and Docker's restart policies
|
|
// key off it, and a zero exit would read as a deliberate stop.
|
|
const StartupFailureExitCode = 1
|
|
|
|
// SentryFlushBudget reports how long the Sentry flush may run when
|
|
// remaining is the time left on the fx stop context after the HTTP
|
|
// drain. sentry.Flush takes a bare duration and honours no context,
|
|
// so this clamp is the only thing keeping a stalled flush from
|
|
// spending the tail hooks' share of the budget on top of a
|
|
// full-length drain. TailHookReserve is held back, and anything
|
|
// under minSentryFlush is skipped rather than attempted uselessly.
|
|
func SentryFlushBudget(remaining time.Duration) time.Duration {
|
|
budget := min(remaining-TailHookReserve, sentryFlushTimeout)
|
|
if budget < minSentryFlush {
|
|
return 0
|
|
}
|
|
|
|
return budget
|
|
}
|
|
|
|
//nolint:revive // ServerParams is a standard fx naming convention.
|
|
type ServerParams struct {
|
|
fx.In
|
|
|
|
Logger *logger.Logger
|
|
Globals *globals.Globals
|
|
Config *config.Config
|
|
Middleware *middleware.Middleware
|
|
Handlers *handlers.Handlers
|
|
Shutdowner fx.Shutdowner
|
|
}
|
|
|
|
// Server is the main HTTP server that wires up routes and manages
|
|
// graceful shutdown.
|
|
type Server struct {
|
|
startupTime time.Time
|
|
|
|
// sentryEnabled is written by the serving goroutine, in
|
|
// enableSentry, and read by the fx stop hook in cleanShutdown.
|
|
// Nothing orders those two: the OnStart hook returns as soon as
|
|
// the goroutine is spawned, so a stop can be running while
|
|
// enableSentry is still deciding. It is atomic to supply the
|
|
// edge the goroutines do not.
|
|
sentryEnabled atomic.Bool
|
|
log *slog.Logger
|
|
cancelFunc context.CancelFunc
|
|
httpServer *http.Server
|
|
router *chi.Mux
|
|
params ServerParams
|
|
mw *middleware.Middleware
|
|
h *handlers.Handlers
|
|
}
|
|
|
|
// New creates a Server that starts the HTTP listener on fx start
|
|
// and stops it gracefully.
|
|
func New(lc fx.Lifecycle, params ServerParams) (*Server, error) {
|
|
s := new(Server)
|
|
s.params = params
|
|
s.mw = params.Middleware
|
|
s.h = params.Handlers
|
|
s.log = params.Logger.Get()
|
|
s.httpServer = s.newHTTPServer()
|
|
|
|
lc.Append(fx.Hook{
|
|
OnStart: func(_ context.Context) error {
|
|
s.startupTime = time.Now()
|
|
go s.Run()
|
|
|
|
return nil
|
|
},
|
|
OnStop: func(ctx context.Context) error {
|
|
s.cleanShutdown(ctx)
|
|
|
|
return nil
|
|
},
|
|
})
|
|
|
|
return s, nil
|
|
}
|
|
|
|
// Run configures Sentry and starts serving HTTP requests.
|
|
//
|
|
// A Sentry failure ends the application instead of listening. It runs
|
|
// before the listener rather than after it so that the process never
|
|
// binds a port it is about to give up.
|
|
func (s *Server) Run() {
|
|
s.configure()
|
|
|
|
// logging before sentry, because sentry logs
|
|
err := s.enableSentry()
|
|
if err != nil {
|
|
s.log.Error(
|
|
"SENTRY_DSN is set but error reporting could not be "+
|
|
"started; refusing to serve with it off",
|
|
"error", err,
|
|
)
|
|
s.shutdownWithFailure()
|
|
|
|
return
|
|
}
|
|
|
|
s.serve()
|
|
}
|
|
|
|
// enableSentry initialises the Sentry SDK when error reporting is
|
|
// configured, and reports the failure when it is configured and cannot
|
|
// be initialised. A DSN that is not set is not a failure: reporting
|
|
// stays off and the server starts normally.
|
|
//
|
|
// There is no fallback to running with reporting off. An operator who
|
|
// set SENTRY_DSN asked for failures to be visible, and serving traffic
|
|
// with reporting quietly off is the one state nothing can ever tell
|
|
// them about — the DSN is still set, so every later signal says it is
|
|
// on. Config already refused a DSN the SDK cannot parse, which is what
|
|
// a typo produces, so reaching this branch means the SDK refused
|
|
// something that parsed: not a condition to guess at either.
|
|
func (s *Server) enableSentry() error {
|
|
s.sentryEnabled.Store(false)
|
|
|
|
if !s.params.Config.SentryEnabled() {
|
|
return nil
|
|
}
|
|
|
|
err := sentry.Init(sentryClientOptions(
|
|
s.params.Config.SentryDSN,
|
|
fmt.Sprintf(
|
|
"%s-%s",
|
|
s.params.Globals.Appname,
|
|
s.params.Globals.Version,
|
|
),
|
|
))
|
|
if err != nil {
|
|
return fmt.Errorf("initialising sentry: %w", err)
|
|
}
|
|
|
|
s.log.Info("sentry error reporting activated")
|
|
s.sentryEnabled.Store(true)
|
|
|
|
return nil
|
|
}
|
|
|
|
// serve installs the signal watcher, starts the listener and blocks
|
|
// until the server's context is cancelled. The process exit status is
|
|
// fx's to decide — from a signal, or from the code
|
|
// shutdownWithFailure hands the Shutdowner — so this reports
|
|
// nothing back to its caller.
|
|
func (s *Server) serve() {
|
|
ctx, cancelFunc := context.WithCancel(context.Background())
|
|
s.cancelFunc = cancelFunc
|
|
|
|
// signal watcher
|
|
go func() {
|
|
c := make(chan os.Signal, 1)
|
|
|
|
signal.Ignore(syscall.SIGPIPE)
|
|
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
|
// block and wait for signal
|
|
sig := <-c
|
|
s.log.Info("signal received", "signal", sig.String())
|
|
|
|
if s.cancelFunc != nil {
|
|
// cancelling the main context will trigger a clean
|
|
// shutdown via the fx OnStop hook.
|
|
s.cancelFunc()
|
|
}
|
|
}()
|
|
|
|
go s.serveUntilShutdown()
|
|
|
|
<-ctx.Done()
|
|
// Shutdown is handled by the fx OnStop hook (cleanShutdown).
|
|
// Do not call cleanShutdown() here to avoid double invocation.
|
|
}
|
|
|
|
// shutdownWithFailure ends the application non-zero from the serving
|
|
// goroutine. It is how anything on that goroutine fails fatally: the
|
|
// fx OnStart hook returns as soon as the goroutine is spawned, so
|
|
// nothing downstream of it ever learns that the goroutine gave up. fx
|
|
// reports RUNNING and the process sits alive having done neither what
|
|
// it was asked nor anything visible instead, which systemd and
|
|
// Docker restart policies cannot see. Asking the Shutdowner to stop
|
|
// the app with a non-zero code is what turns that into a visible
|
|
// failure, and it is the whole of "fatal" here — no panic, no
|
|
// os.Exit, and every stop hook still runs.
|
|
//
|
|
// The context cancel that follows only unwinds serve()'s own wait,
|
|
// and is skipped before serve has installed one. The shutdown itself
|
|
// runs through fx's normal stop sequence, so the clean-shutdown drain
|
|
// in cleanShutdown is reached unchanged.
|
|
func (s *Server) shutdownWithFailure() {
|
|
err := s.params.Shutdowner.Shutdown(
|
|
fx.ExitCode(StartupFailureExitCode),
|
|
)
|
|
if err != nil {
|
|
s.log.Error("shutdown request failed", "error", err)
|
|
}
|
|
|
|
if s.cancelFunc != nil {
|
|
s.cancelFunc()
|
|
}
|
|
}
|
|
|
|
func (s *Server) cleanupForExit() {
|
|
s.log.Info("cleaning up")
|
|
}
|
|
|
|
func (s *Server) cleanShutdown(ctx context.Context) {
|
|
ctxShutdown, shutdownCancel := context.WithTimeout(
|
|
ctx, ShutdownTimeout,
|
|
)
|
|
defer shutdownCancel()
|
|
|
|
err := s.httpServer.Shutdown(ctxShutdown)
|
|
if err != nil {
|
|
s.log.Error(
|
|
"server clean shutdown failed", "error", err,
|
|
)
|
|
}
|
|
|
|
s.cleanupForExit()
|
|
|
|
if s.sentryEnabled.Load() {
|
|
s.flushSentry(ctx)
|
|
}
|
|
}
|
|
|
|
// flushSentry drains Sentry's queue inside what is left of the fx
|
|
// stop budget. A context carrying no deadline — a caller outside the
|
|
// fx lifecycle — gets the full timeout.
|
|
func (s *Server) flushSentry(ctx context.Context) {
|
|
flush := sentryFlushTimeout
|
|
|
|
if deadline, ok := ctx.Deadline(); ok {
|
|
flush = SentryFlushBudget(time.Until(deadline))
|
|
}
|
|
|
|
if flush <= 0 {
|
|
s.log.Warn(
|
|
"skipping sentry flush, stop budget exhausted",
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
sentry.Flush(flush)
|
|
}
|
|
|
|
func (s *Server) configure() {
|
|
// identify ourselves in the logs
|
|
s.params.Logger.Identify()
|
|
}
|