Compare commits
2 Commits
50a49e8d4e
...
d333572592
| Author | SHA1 | Date | |
|---|---|---|---|
| d333572592 | |||
| bef9986542 |
@@ -13,39 +13,19 @@ jobs:
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
||||
with:
|
||||
# The fingerprint step below needs history to find the last commit
|
||||
# that touched the Docker build context.
|
||||
# that touched the Docker build context, and the superseded-status
|
||||
# step needs it to walk ancestors (it aborts on a shallow clone).
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Neutralize superseded run statuses
|
||||
- name: Mark superseded run statuses
|
||||
# Gitea cancels the in-flight run when another commit is pushed to the
|
||||
# same branch and records the cancellation as `failure`, so a commit
|
||||
# that was never tested reads red. The cancellation is unconditional
|
||||
# server-side for push events and cannot be disabled from a workflow
|
||||
# file, so the superseding run rewrites those statuses to `skipped`.
|
||||
# Only the exact cancellation status is touched; a real failure is
|
||||
# left alone.
|
||||
# that was never tested reads as a test result. The script rewrites
|
||||
# those statuses to say what happened. See its header for why the
|
||||
# state stays `failure` and not `skipped`.
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
api="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}"
|
||||
ctx='check / check (push)'
|
||||
for sha in $(git rev-list --max-count=20 "${GITHUB_SHA}^" || true); do
|
||||
latest="$(curl -sf "${api}/commits/${sha}/status" | jq -r \
|
||||
--arg c "$ctx" \
|
||||
'[.statuses[] | select(.context == $c)][0] // empty
|
||||
| "\(.status)|\(.description)"')" || continue
|
||||
[ "$latest" = 'failure|Has been cancelled' ] || continue
|
||||
curl -sf -X POST "${api}/statuses/${sha}" \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "$(jq -nc --arg c "$ctx" '{
|
||||
context: $c,
|
||||
state: "skipped",
|
||||
description: "Superseded by a newer commit; never tested"
|
||||
}')" >/dev/null
|
||||
echo "neutralized superseded status on ${sha}"
|
||||
done
|
||||
run: script/ci-mark-superseded
|
||||
|
||||
- name: Fingerprint the build context
|
||||
# `.dockerignore` keeps docs out of the build context, so a docs-only
|
||||
|
||||
@@ -32,7 +32,9 @@ FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a349228
|
||||
# Depend on lint stage passing
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates && rm -rf /var/lib/apt/lists/*
|
||||
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
||||
# suite executes.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates jq && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
|
||||
109
README.md
109
README.md
@@ -282,6 +282,8 @@ are inline commands with no script behind them. We provide:
|
||||
- `script/docker` — build the Docker image tagged via `script/projectname`
|
||||
- `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile
|
||||
runs the checks, so a green build implies a green repo)
|
||||
- `script/ci-mark-superseded` — CI helper: mark the commits whose run a
|
||||
newer push cancelled (see [CI gate honesty](#ci-gate-honesty))
|
||||
- `script/precommit` — pre-commit checks (`go mod tidy` guard, then
|
||||
`script/check`)
|
||||
- `script/install-precommit` — install the git pre-commit hook that
|
||||
@@ -1145,8 +1147,6 @@ webhooker/
|
||||
│ │ ├── archive_sweeper.go # Periodic pruning of idle archives
|
||||
│ │ ├── url_mask.go # Strips credentials from *url.Error
|
||||
│ │ └── ssrf.go # SSRF prevention (IP validation, safe HTTP transport)
|
||||
│ ├── lifecycle/
|
||||
│ │ └── lifecycle.go # Shared fx start/stop hook helpers
|
||||
│ ├── handlers/
|
||||
│ │ ├── handlers.go # Base handler struct, JSON helpers, template rendering
|
||||
│ │ ├── auth.go # Login, logout handlers
|
||||
@@ -1158,6 +1158,8 @@ webhooker/
|
||||
│ │ └── webhook.go # Webhook receiver handler
|
||||
│ ├── healthcheck/
|
||||
│ │ └── healthcheck.go # Health check service (uptime, version)
|
||||
│ ├── lifecycle/
|
||||
│ │ └── lifecycle.go # Shared stop-hook waiter, bounded by the stop context
|
||||
│ ├── logger/
|
||||
│ │ └── logger.go # slog setup with TTY detection
|
||||
│ ├── middleware/
|
||||
@@ -1316,6 +1318,78 @@ rather than global: **LoginRateLimit** on `/pages/login`,
|
||||
- GORM soft deletes on every entity that carries `BaseModel`, which is
|
||||
all of them but `Setting` (data preserved for audit)
|
||||
|
||||
### Shutdown
|
||||
|
||||
On SIGINT or SIGTERM, fx runs the registered stop hooks in reverse
|
||||
dependency order under a **5 second budget** (`fx.StopTimeout` in
|
||||
`cmd/webhooker/main.go`). That budget covers the whole sequence, not
|
||||
each hook. The order, read off the fx stop-hook log:
|
||||
|
||||
1. `ArchiveSweeper`
|
||||
2. `RetentionReaper`
|
||||
3. `server` — the HTTP drain, bounded separately by
|
||||
`server.ShutdownTimeout` (**3 seconds**), then a Sentry flush if
|
||||
`SENTRY_DSN` is set
|
||||
4. `delivery.Engine`
|
||||
5. `healthcheck`
|
||||
6. `WebhookDBManager`
|
||||
7. the database close
|
||||
|
||||
The two components that can realistically hold the budget run
|
||||
first: a retention sweep or an archive prune caught mid-tick each
|
||||
waits on its `WaitGroup` bounded by the stop context, so a wedge
|
||||
there consumes the 5 seconds before the HTTP server hook is ever
|
||||
entered. The hooks after the server are microsecond-scale in normal
|
||||
operation.
|
||||
|
||||
The HTTP drain budget is deliberately **shorter** than the sequence
|
||||
budget. Were the two equal, a drain that used its whole budget would
|
||||
exhaust the sequence budget at the instant it finished, and every
|
||||
later hook — the delivery engine, the healthcheck, the webhook DB
|
||||
manager and the database close — would be skipped in exactly the
|
||||
case where the drain mattered. 3 seconds leaves 2 seconds
|
||||
(`server.TailHookReserve`) for the tail, which is far more than the
|
||||
microseconds it needs.
|
||||
|
||||
That reserve belongs to the tail hooks, not to the server hook, and
|
||||
the Sentry flush is what could take it: it runs after the drain
|
||||
**inside the same hook**, and `sentry.Flush` takes a bare duration
|
||||
and honours no context, so an unreachable Sentry endpoint would add
|
||||
its own timeout on top of a full-length drain and consume the whole
|
||||
sequence budget by itself. It is therefore clamped to whatever is
|
||||
left on the stop context minus the reserve, and skipped when that
|
||||
leaves too little to be worth attempting — so a full-length drain
|
||||
means Sentry events are dropped rather than the database close being
|
||||
skipped.
|
||||
|
||||
This does not make the database close unconditional: a wedged
|
||||
`ArchiveSweeper` or `RetentionReaper` still runs first and can
|
||||
consume the whole budget on its own.
|
||||
|
||||
The value is chosen to sit inside the container stop grace period.
|
||||
Docker's default `docker stop` grace is 10 seconds and the Dockerfile
|
||||
sets no `STOPSIGNAL` or grace override, so the process must be gone
|
||||
before that. fx's own default is 15 seconds, which is past the grace:
|
||||
the container would be SIGKILLed (exit 137) before the bound could
|
||||
fire, and nothing that depends on it — including the
|
||||
`shutdown timed out, goroutines still running` error log that tells
|
||||
an operator a component is wedged — would ever be reached.
|
||||
|
||||
Two operational consequences follow from bounding the sequence:
|
||||
|
||||
- **A wedged component aborts the rest of the shutdown.** fx checks
|
||||
the stop context before each remaining hook and returns outright
|
||||
once it has expired, skipping the hooks it has not reached. If the
|
||||
first-stopped component consumes the whole budget, the later hooks
|
||||
never run — **the database close among them**. SQLite is crash-safe,
|
||||
so this is not corruption, but it is not a clean close either.
|
||||
- **Lowering the grace below 5 seconds reintroduces the silent
|
||||
truncation.** `docker stop --time`, Compose's `stop_grace_period`,
|
||||
or Kubernetes' `terminationGracePeriodSeconds` set under 5 seconds
|
||||
put SIGKILL back in front of the bound, and the process dies with
|
||||
no shutdown diagnostics at all. Keep the deployment's grace above
|
||||
the stop timeout.
|
||||
|
||||
### Docker
|
||||
|
||||
The Dockerfile uses a three-stage build. Each stage is pinned by
|
||||
@@ -1373,11 +1447,32 @@ way.
|
||||
A separate workflow step, run before the fingerprint is written, covers
|
||||
a second way the gate lied: Gitea cancels an in-flight run when a newer
|
||||
commit lands on the same branch and records that cancellation as a
|
||||
`failure` status, marking a commit red that was never tested.
|
||||
Cancellation is unconditional server-side for
|
||||
push events, so the superseding run rewrites the exact
|
||||
`Has been cancelled` status to `skipped`. Genuine failures are never
|
||||
touched.
|
||||
`failure` status, so a commit nothing ever tested reads as a test
|
||||
result. Cancellation is unconditional server-side for push events, so
|
||||
the superseding run calls `script/ci-mark-superseded`, which rewrites
|
||||
that exact status to `failure` /
|
||||
`Superseded by a newer commit; never tested`.
|
||||
|
||||
The state stays `failure` on purpose: Gitea's combined status folds
|
||||
`skipped` into `success`, so marking a never-tested commit `skipped`
|
||||
made the status API report green for it, indistinguishable from a commit
|
||||
that passed. Reading a commit's status on this repo therefore goes:
|
||||
|
||||
- `success` / `Successful in ...` — the checks ran and passed.
|
||||
- `failure` / `Failing after ...` — the checks ran and failed.
|
||||
- `failure` / `Superseded by a newer commit; never tested` — the run was
|
||||
cancelled, by a newer push or by hand, and nothing was verified about
|
||||
this commit. Test the commit itself before concluding anything about
|
||||
it.
|
||||
|
||||
Genuine failures and successes are never touched, and no status is left
|
||||
`pending`, which would block the commit indefinitely. The step derives
|
||||
its context string from the workflow name, the job **id** and the event.
|
||||
That is deliberately not byte-identical to Gitea's own rule, which uses
|
||||
the job's display `name:` where the runner exports the id, so giving the
|
||||
job a `name:` — or renaming the workflow — makes the derived context
|
||||
stop matching. The step fails loudly when no status on the commit
|
||||
carries that context, so no rename can silently disable the rewrite.
|
||||
|
||||
## TODO
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
@@ -15,6 +17,33 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
|
||||
// stopTimeout bounds the whole fx stop sequence, not each hook.
|
||||
//
|
||||
// fx defaults to 15s, which is longer than Docker's 10s default
|
||||
// stop grace: the container would be SIGKILLed before the bound
|
||||
// could fire, so nothing bounded by it would ever be observed.
|
||||
// 5s leaves headroom inside that grace for signal delivery and
|
||||
// process exit; the observed wedge case already exits at ~5.3s,
|
||||
// so a larger bound would trade a rare skipped database close for
|
||||
// a more common hard kill.
|
||||
//
|
||||
// The server's stop hook must fit inside it with room to spare: a
|
||||
// hook that used the whole budget would exhaust it at that instant,
|
||||
// and fx would skip every hook after the server — the delivery
|
||||
// engine, the healthcheck, the webhook DB manager and the database
|
||||
// close. That hook is the 3s HTTP drain plus the Sentry flush that
|
||||
// follows it in the same hook, so the flush is clamped to the stop
|
||||
// context's remaining time less server.TailHookReserve rather than
|
||||
// running for its own fixed 2s; the reserve is what the tail hooks
|
||||
// live on, and they are microsecond-scale in normal operation.
|
||||
// TestStopTimeout_LeavesHeadroomForTailHooks pins the arithmetic
|
||||
// across every drain length.
|
||||
//
|
||||
// This does not make the database close unconditional: the
|
||||
// ArchiveSweeper and RetentionReaper hooks run before the server
|
||||
// and can still consume the whole budget on their own.
|
||||
const stopTimeout = 5 * time.Second
|
||||
|
||||
// Build-time variables set via -ldflags.
|
||||
//
|
||||
//nolint:gochecknoglobals // Build-time variables injected by the linker.
|
||||
@@ -27,7 +56,14 @@ func main() {
|
||||
globals.Appname = appname
|
||||
globals.Version = version
|
||||
|
||||
fx.New(
|
||||
newApp().Run()
|
||||
}
|
||||
|
||||
// newApp builds the application graph. It is separate from main so
|
||||
// a test can assert the options it carries.
|
||||
func newApp() *fx.App {
|
||||
return fx.New(
|
||||
fx.StopTimeout(stopTimeout),
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
@@ -60,5 +96,5 @@ func main() {
|
||||
) {
|
||||
},
|
||||
),
|
||||
).Run()
|
||||
)
|
||||
}
|
||||
|
||||
75
cmd/webhooker/main_test.go
Normal file
75
cmd/webhooker/main_test.go
Normal file
@@ -0,0 +1,75 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/server"
|
||||
)
|
||||
|
||||
// dockerStopGrace is Docker's default `docker stop` grace period.
|
||||
// The Dockerfile sets no STOPSIGNAL or grace override, so this is
|
||||
// the deadline the container is actually held to, and the fx stop
|
||||
// timeout has to fit inside it with room for signal delivery and
|
||||
// process exit.
|
||||
const dockerStopGrace = 10 * time.Second
|
||||
|
||||
// TestNewApp_StopTimeout pins the fx stop timeout. Without the
|
||||
// explicit fx.StopTimeout option the app reads fx's 15s
|
||||
// DefaultTimeout, which exceeds dockerStopGrace: the container is
|
||||
// SIGKILLed before the bound fires and every shutdown hook bounded
|
||||
// by it — including the operator-facing timeout log — becomes
|
||||
// unreachable in the image this repo produces.
|
||||
//
|
||||
// fx.New applies options before it executes invokes, so the timeout
|
||||
// is set whether or not the graph itself can be constructed here.
|
||||
func TestNewApp_StopTimeout(t *testing.T) {
|
||||
t.Setenv("DATA_DIR", t.TempDir())
|
||||
|
||||
got := newApp().StopTimeout()
|
||||
|
||||
require.Equal(t, stopTimeout, got)
|
||||
require.Less(t, got, dockerStopGrace)
|
||||
}
|
||||
|
||||
// tailHeadroom is the slack the fx stop budget must keep beyond the
|
||||
// server stop hook. The hooks that run after the server — the
|
||||
// delivery engine, the healthcheck, the webhook DB manager and the
|
||||
// database close — are microsecond-scale in normal operation, so
|
||||
// this is generous for them.
|
||||
const tailHeadroom = 2 * time.Second
|
||||
|
||||
// TestStopTimeout_LeavesHeadroomForTailHooks pins the relationship
|
||||
// between the server's stop hook and the fx stop budget. fx bounds
|
||||
// the whole stop sequence, and returns without running its
|
||||
// remaining hooks once the stop context has expired. If the hook
|
||||
// could use the entire budget, every later hook — the database close
|
||||
// included — would be skipped in exactly the case where the drain
|
||||
// mattered.
|
||||
//
|
||||
// The hook is not just the HTTP drain: a Sentry flush follows it in
|
||||
// the same hook, and sentry.Flush honours no context, so both halves
|
||||
// have to be counted. The sweep walks every drain length the hook
|
||||
// can produce, since a shorter drain leaves the flush more room and
|
||||
// the worst case is not necessarily at either extreme.
|
||||
//
|
||||
// Shrinking either budget, or unbounding the flush again, must fail
|
||||
// here rather than silently recreating a hook that swallows the
|
||||
// whole sequence.
|
||||
func TestStopTimeout_LeavesHeadroomForTailHooks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
require.Less(t, server.ShutdownTimeout, stopTimeout)
|
||||
|
||||
const step = 10 * time.Millisecond
|
||||
|
||||
for drain := time.Duration(0); drain <= server.ShutdownTimeout; drain += step {
|
||||
hook := drain + server.SentryFlushBudget(stopTimeout-drain)
|
||||
|
||||
require.LessOrEqual(
|
||||
t, hook+tailHeadroom, stopTimeout,
|
||||
"a %s drain leaves the tail hooks short", drain,
|
||||
)
|
||||
}
|
||||
}
|
||||
2
go.mod
2
go.mod
@@ -17,6 +17,7 @@ require (
|
||||
github.com/stretchr/testify v1.8.4
|
||||
go.uber.org/fx v1.20.1
|
||||
golang.org/x/crypto v0.38.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gorm.io/driver/sqlite v1.5.4
|
||||
gorm.io/gorm v1.25.5
|
||||
modernc.org/sqlite v1.28.0
|
||||
@@ -52,7 +53,6 @@ require (
|
||||
golang.org/x/text v0.25.0 // indirect
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||
google.golang.org/protobuf v1.31.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/uint128 v1.2.0 // indirect
|
||||
modernc.org/cc/v3 v3.40.0 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||
|
||||
387
internal/ciscript/ci_mark_superseded_test.go
Normal file
387
internal/ciscript/ci_mark_superseded_test.go
Normal file
@@ -0,0 +1,387 @@
|
||||
package ciscript_test
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
// supersededDesc is the description script/ci-mark-superseded
|
||||
// writes, and the one an earlier revision of it wrote alongside a
|
||||
// `skipped` state.
|
||||
supersededDesc = "Superseded by a newer commit; never tested"
|
||||
|
||||
// liveContext is the commit-status context Gitea uses for this
|
||||
// repository's runs, as seen in its API. The script derives it from
|
||||
// the workflow and job names rather than hardcoding it; the
|
||||
// derivation is checked against this value below.
|
||||
liveContext = "check / check (push)"
|
||||
|
||||
scriptPath = "../../script/ci-mark-superseded"
|
||||
workflow = "../../.gitea/workflows/check.yml"
|
||||
|
||||
// failure is the only state that neither folds into a combined
|
||||
// `success` (as `skipped` does) nor blocks the commit forever (as
|
||||
// `pending` does).
|
||||
failure = "failure"
|
||||
)
|
||||
|
||||
// repo is a throwaway git history: parent is the commit a run would be
|
||||
// cancelled on, head the commit that superseded it.
|
||||
type repo struct {
|
||||
dir string
|
||||
head string
|
||||
parent string
|
||||
}
|
||||
|
||||
// scriptEnv is the run identity the Gitea runner exports and the script
|
||||
// builds its context string from.
|
||||
type scriptEnv struct {
|
||||
workflow string
|
||||
job string
|
||||
event string
|
||||
}
|
||||
|
||||
func defaultEnv() scriptEnv {
|
||||
return scriptEnv{workflow: "check", job: "check", event: "push"}
|
||||
}
|
||||
|
||||
func cancelled() commitStatus {
|
||||
return commitStatus{
|
||||
Context: liveContext,
|
||||
Status: failure,
|
||||
Description: "Has been cancelled",
|
||||
}
|
||||
}
|
||||
|
||||
func running() commitStatus {
|
||||
return commitStatus{
|
||||
Context: liveContext,
|
||||
Status: "pending",
|
||||
Description: "Has started running",
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkSuperseded(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := map[string]struct {
|
||||
parent commitStatus
|
||||
wantMark bool
|
||||
}{
|
||||
"a cancelled run is marked": {
|
||||
parent: cancelled(),
|
||||
wantMark: true,
|
||||
},
|
||||
"a laundered skipped status is marked": {
|
||||
parent: commitStatus{
|
||||
Context: liveContext,
|
||||
Status: "skipped",
|
||||
Description: supersededDesc,
|
||||
},
|
||||
wantMark: true,
|
||||
},
|
||||
"a genuine failure is left alone": {
|
||||
parent: commitStatus{
|
||||
Context: liveContext,
|
||||
Status: failure,
|
||||
Description: "Failing after 3m1s",
|
||||
},
|
||||
wantMark: false,
|
||||
},
|
||||
"a passing run is left alone": {
|
||||
parent: commitStatus{
|
||||
Context: liveContext,
|
||||
Status: "success",
|
||||
Description: "Successful in 2m52s",
|
||||
},
|
||||
wantMark: false,
|
||||
},
|
||||
"another context is left alone": {
|
||||
parent: commitStatus{
|
||||
Context: "other / other (push)",
|
||||
Status: failure,
|
||||
Description: "Has been cancelled",
|
||||
},
|
||||
wantMark: false,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireTools(t)
|
||||
|
||||
history := newRepo(t)
|
||||
fake, api := newFakeGitea(t)
|
||||
fake.setStatus(history.head, running())
|
||||
fake.setStatus(history.parent, tc.parent)
|
||||
|
||||
out, err := runScript(t, history, api, defaultEnv())
|
||||
require.NoError(t, err, out)
|
||||
|
||||
posted := fake.postedFor(history.parent)
|
||||
if !tc.wantMark {
|
||||
require.Empty(t, posted)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
require.Equal(t, []postedStatus{{
|
||||
Context: liveContext,
|
||||
// Not `skipped`: Gitea's combined status folds
|
||||
// that into `success`, which is what made a
|
||||
// never-tested commit read green.
|
||||
State: failure,
|
||||
Description: supersededDesc,
|
||||
}}, posted)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A second run must not rewrite what the first one wrote, or every
|
||||
// later push would post a duplicate status.
|
||||
func TestMarkSupersededIsIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireTools(t)
|
||||
|
||||
history := newRepo(t)
|
||||
fake, api := newFakeGitea(t)
|
||||
fake.setStatus(history.head, running())
|
||||
fake.setStatus(history.parent, cancelled())
|
||||
|
||||
for range 2 {
|
||||
out, err := runScript(t, history, api, defaultEnv())
|
||||
require.NoError(t, err, out)
|
||||
}
|
||||
|
||||
require.Len(t, fake.postedFor(history.parent), 1)
|
||||
}
|
||||
|
||||
// Renaming the workflow or the job changes the context string Gitea
|
||||
// uses. The script must say so instead of quietly matching nothing.
|
||||
func TestMarkSupersededRejectsAnUnknownContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireTools(t)
|
||||
|
||||
history := newRepo(t)
|
||||
fake, api := newFakeGitea(t)
|
||||
fake.setStatus(history.head, running())
|
||||
fake.setStatus(history.parent, cancelled())
|
||||
|
||||
env := defaultEnv()
|
||||
env.job = "renamed"
|
||||
|
||||
out, err := runScript(t, history, api, env)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, out, "renamed")
|
||||
require.Contains(t, out, liveContext)
|
||||
require.Empty(t, fake.postedFor(history.parent))
|
||||
}
|
||||
|
||||
// ANCESTOR_LIMIT is a documented knob. A value that is set but unusable
|
||||
// must abort: handing it to git and discarding the exit status left the
|
||||
// walk empty and the step green, marking nothing.
|
||||
func TestMarkSupersededRejectsAnUnparseableAncestorLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireTools(t)
|
||||
|
||||
history := newRepo(t)
|
||||
fake, api := newFakeGitea(t)
|
||||
fake.setStatus(history.head, running())
|
||||
fake.setStatus(history.parent, cancelled())
|
||||
|
||||
out, err := runScript(
|
||||
t, history, api, defaultEnv(), "ANCESTOR_LIMIT=twenty",
|
||||
)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, out, "ANCESTOR_LIMIT")
|
||||
require.Contains(t, out, "twenty")
|
||||
require.Empty(t, fake.postedFor(history.parent))
|
||||
}
|
||||
|
||||
// A status read that fails is not the same as a commit with nothing to
|
||||
// do. Losing curl's exit status through a pipe made the two identical
|
||||
// and left a laundered commit laundered with no signal.
|
||||
func TestMarkSupersededFailsOnAnUnreadableAncestorStatus(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireTools(t)
|
||||
|
||||
history := newRepo(t)
|
||||
fake, api := newFakeGitea(t)
|
||||
fake.setStatus(history.head, running())
|
||||
fake.setStatus(history.parent, cancelled())
|
||||
fake.failStatusRead(history.parent)
|
||||
|
||||
out, err := runScript(t, history, api, defaultEnv())
|
||||
require.Error(t, err)
|
||||
require.Contains(t, out, history.parent)
|
||||
require.Contains(t, out, "cannot read commit statuses")
|
||||
require.Empty(t, fake.postedFor(history.parent))
|
||||
}
|
||||
|
||||
// A shallow clone cannot resolve the parent, so it is indistinguishable
|
||||
// from a root commit to rev-parse and the walk would exit 0 having
|
||||
// marked nothing. It must abort instead: dropping `fetch-depth: 0` from
|
||||
// the checkout step is one edit, and a silent no-op there restores the
|
||||
// false-green bug this script exists to prevent.
|
||||
func TestMarkSupersededRejectsAShallowRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireTools(t)
|
||||
|
||||
history := shallowClone(t, newRepo(t))
|
||||
fake, api := newFakeGitea(t)
|
||||
fake.setStatus(history.head, running())
|
||||
fake.setStatus(history.parent, cancelled())
|
||||
|
||||
out, err := runScript(t, history, api, defaultEnv())
|
||||
require.Error(t, err)
|
||||
require.Contains(t, out, "shallow repository")
|
||||
require.Empty(t, fake.postedFor(history.parent))
|
||||
require.Empty(t, fake.postedFor(history.head))
|
||||
}
|
||||
|
||||
// shallowClone returns the same history as a depth-1 clone. The `file://`
|
||||
// URL is required: git ignores --depth for a plain local path.
|
||||
func shallowClone(t *testing.T, history repo) repo {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
|
||||
//nolint:gosec // fixed argv, arguments are test-local paths
|
||||
cmd := exec.CommandContext(t.Context(), "git", "clone", "-q",
|
||||
"--depth=1", "file://"+history.dir, dir)
|
||||
|
||||
out, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, string(out))
|
||||
|
||||
return repo{dir: dir, head: history.head, parent: history.parent}
|
||||
}
|
||||
|
||||
// The derived context must equal the one Gitea actually uses, which is
|
||||
// built from the same workflow and job names.
|
||||
func TestDerivedContextMatchesGitea(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireTools(t)
|
||||
|
||||
name, job := workflowIdentity(t)
|
||||
|
||||
history := newRepo(t)
|
||||
fake, api := newFakeGitea(t)
|
||||
fake.setStatus(history.head, running())
|
||||
fake.setStatus(history.parent, cancelled())
|
||||
|
||||
out, err := runScript(t, history, api, scriptEnv{
|
||||
workflow: name,
|
||||
job: job,
|
||||
event: "push",
|
||||
})
|
||||
require.NoError(t, err, out)
|
||||
|
||||
posted := fake.postedFor(history.parent)
|
||||
require.Len(t, posted, 1)
|
||||
require.Equal(t, liveContext, posted[0].Context)
|
||||
}
|
||||
|
||||
// workflowIdentity reads the workflow name and its single job id out of
|
||||
// the checked-in workflow file.
|
||||
func workflowIdentity(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
raw, err := os.ReadFile(workflow)
|
||||
require.NoError(t, err)
|
||||
|
||||
var parsed struct {
|
||||
Name string `yaml:"name"`
|
||||
Jobs map[string]any `yaml:"jobs"`
|
||||
}
|
||||
|
||||
require.NoError(t, yaml.Unmarshal(raw, &parsed))
|
||||
|
||||
jobs := slices.Collect(maps.Keys(parsed.Jobs))
|
||||
require.Len(t, jobs, 1)
|
||||
|
||||
return parsed.Name, jobs[0]
|
||||
}
|
||||
|
||||
func runScript(
|
||||
t *testing.T, history repo, api string, env scriptEnv,
|
||||
extra ...string,
|
||||
) (string, error) {
|
||||
t.Helper()
|
||||
|
||||
script, err := filepath.Abs(scriptPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
//nolint:gosec // fixed argv, repo-local script under test
|
||||
cmd := exec.CommandContext(t.Context(), "sh", script)
|
||||
cmd.Dir = history.dir
|
||||
cmd.Env = append(os.Environ(),
|
||||
"GITHUB_API_URL="+api,
|
||||
"GITHUB_REPOSITORY=sneak/webhooker",
|
||||
"GITHUB_SHA="+history.head,
|
||||
"GITHUB_WORKFLOW="+env.workflow,
|
||||
"GITHUB_JOB="+env.job,
|
||||
"GITHUB_EVENT_NAME="+env.event,
|
||||
"GITEA_TOKEN=test-token",
|
||||
)
|
||||
cmd.Env = append(cmd.Env, extra...)
|
||||
|
||||
out, err := cmd.CombinedOutput()
|
||||
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
func newRepo(t *testing.T) repo {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
|
||||
git := func(args ...string) string {
|
||||
//nolint:gosec // fixed argv, arguments are test constants
|
||||
cmd := exec.CommandContext(t.Context(), "git", args...)
|
||||
cmd.Dir = dir
|
||||
|
||||
out, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, string(out))
|
||||
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
commit := func(message string) string {
|
||||
git(
|
||||
"-c", "user.email=ci@example.invalid",
|
||||
"-c", "user.name=ci",
|
||||
"-c", "commit.gpgsign=false",
|
||||
"commit", "-q", "--allow-empty", "-m", message,
|
||||
)
|
||||
|
||||
return git("rev-parse", "HEAD")
|
||||
}
|
||||
|
||||
git("init", "-q", "-b", "main")
|
||||
|
||||
parent := commit("parent")
|
||||
head := commit("head")
|
||||
|
||||
return repo{dir: dir, head: head, parent: parent}
|
||||
}
|
||||
|
||||
func requireTools(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
for _, tool := range []string{"sh", "git", "curl", "jq"} {
|
||||
_, err := exec.LookPath(tool)
|
||||
if err != nil {
|
||||
t.Skipf("%s is not installed: %v", tool, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
10
internal/ciscript/doc.go
Normal file
10
internal/ciscript/doc.go
Normal file
@@ -0,0 +1,10 @@
|
||||
// Package ciscript holds the tests for the repository's CI shell
|
||||
// scripts in script/. It carries no runtime code: the scripts run on
|
||||
// the CI runner, not inside the binary, but their behaviour still has
|
||||
// to be verified by the test suite.
|
||||
//
|
||||
// The scripts under test are outside the Go build graph, so `go test`'s
|
||||
// result cache serves a stale PASS when only a script changed: run the
|
||||
// container build, or GOFLAGS=-count=1, to trust a result here after
|
||||
// editing script/.
|
||||
package ciscript
|
||||
162
internal/ciscript/fakegitea_test.go
Normal file
162
internal/ciscript/fakegitea_test.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package ciscript_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// commitStatus is the part of an entry in Gitea's combined-status
|
||||
// response that script/ci-mark-superseded reads.
|
||||
type commitStatus struct {
|
||||
Context string `json:"context"`
|
||||
Status string `json:"status"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
// postedStatus is the part of a create-status request body the script
|
||||
// writes.
|
||||
type postedStatus struct {
|
||||
Context string `json:"context"`
|
||||
State string `json:"state"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
// fakeGitea serves the two endpoints the script talks to. Like Gitea,
|
||||
// the newest status for a context replaces the previous one, so a
|
||||
// second run of the script sees what the first one wrote.
|
||||
type fakeGitea struct {
|
||||
mu sync.Mutex
|
||||
statuses map[string][]commitStatus
|
||||
posted map[string][]postedStatus
|
||||
// failRead is a commit whose combined-status read answers HTTP
|
||||
// 500, standing in for a status API that is down.
|
||||
failRead string
|
||||
}
|
||||
|
||||
// newFakeGitea returns the fake and the base URL to hand the script as
|
||||
// GITHUB_API_URL.
|
||||
func newFakeGitea(t *testing.T) (*fakeGitea, string) {
|
||||
t.Helper()
|
||||
|
||||
fake := &fakeGitea{
|
||||
mu: sync.Mutex{},
|
||||
statuses: map[string][]commitStatus{},
|
||||
posted: map[string][]postedStatus{},
|
||||
failRead: "",
|
||||
}
|
||||
|
||||
srv := httptest.NewServer(fake.routes())
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
return fake, srv.URL
|
||||
}
|
||||
|
||||
func (f *fakeGitea) routes() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc(
|
||||
"GET /repos/{owner}/{repo}/commits/{sha}/status",
|
||||
f.handleCombined,
|
||||
)
|
||||
mux.HandleFunc(
|
||||
"POST /repos/{owner}/{repo}/statuses/{sha}",
|
||||
f.handleCreate,
|
||||
)
|
||||
|
||||
return mux
|
||||
}
|
||||
|
||||
func (f *fakeGitea) handleCombined(
|
||||
w http.ResponseWriter, r *http.Request,
|
||||
) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
sha := r.PathValue("sha")
|
||||
if f.failRead != "" && f.failRead == sha {
|
||||
http.Error(w, "boom", http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
body := struct {
|
||||
Statuses []commitStatus `json:"statuses"`
|
||||
}{Statuses: f.statuses[sha]}
|
||||
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
_, _ = w.Write(payload)
|
||||
}
|
||||
|
||||
func (f *fakeGitea) handleCreate(w http.ResponseWriter, r *http.Request) {
|
||||
var got postedStatus
|
||||
|
||||
err := json.NewDecoder(r.Body).Decode(&got)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
sha := r.PathValue("sha")
|
||||
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
f.posted[sha] = append(f.posted[sha], got)
|
||||
f.replaceLocked(sha, commitStatus{
|
||||
Context: got.Context,
|
||||
Status: got.State,
|
||||
Description: got.Description,
|
||||
})
|
||||
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
|
||||
// failStatusRead makes the combined-status read for one commit answer
|
||||
// HTTP 500.
|
||||
func (f *fakeGitea) failStatusRead(sha string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
f.failRead = sha
|
||||
}
|
||||
|
||||
// setStatus gives a commit its latest status for a context.
|
||||
func (f *fakeGitea) setStatus(sha string, status commitStatus) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
f.replaceLocked(sha, status)
|
||||
}
|
||||
|
||||
// postedFor returns the statuses the script created for a commit.
|
||||
func (f *fakeGitea) postedFor(sha string) []postedStatus {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
return append([]postedStatus(nil), f.posted[sha]...)
|
||||
}
|
||||
|
||||
// replaceLocked requires f.mu.
|
||||
func (f *fakeGitea) replaceLocked(sha string, status commitStatus) {
|
||||
for i, existing := range f.statuses[sha] {
|
||||
if existing.Context == status.Context {
|
||||
f.statuses[sha][i] = status
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
f.statuses[sha] = append(f.statuses[sha], status)
|
||||
}
|
||||
21
internal/lifecycle/export_test.go
Normal file
21
internal/lifecycle/export_test.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// WaitDone exposes waitDone to the external test package. Only the
|
||||
// unexported waiter can be handed a channel that is already closed
|
||||
// before the call, which is the state the preamble exists for;
|
||||
// through WaitForShutdown the waiter goroutine may or may not have
|
||||
// closed the channel yet, so the case is not reachable
|
||||
// deterministically from outside.
|
||||
func WaitDone(
|
||||
ctx context.Context,
|
||||
log *slog.Logger,
|
||||
component string,
|
||||
done <-chan struct{},
|
||||
) error {
|
||||
return waitDone(ctx, log, component, done)
|
||||
}
|
||||
@@ -38,6 +38,29 @@ func WaitForShutdown(
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
return waitDone(ctx, log, component, done)
|
||||
}
|
||||
|
||||
// waitDone waits for done to close, bounded by ctx.
|
||||
//
|
||||
// The non-blocking preamble is load-bearing. When the component has
|
||||
// already drained and ctx has already expired, both cases of the
|
||||
// bounded select are ready and Go picks between them uniformly at
|
||||
// random, so a clean shutdown would be reported as a timeout about
|
||||
// half the time. Draining wins: the goroutines are gone, and there
|
||||
// is nothing left for the operator to act on.
|
||||
func waitDone(
|
||||
ctx context.Context,
|
||||
log *slog.Logger,
|
||||
component string,
|
||||
done <-chan struct{},
|
||||
) error {
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
|
||||
@@ -37,6 +37,57 @@ func TestWaitForShutdown_DrainedGroup(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
// racePasses is how many times the both-cases-ready race is run.
|
||||
// Without the preamble each pass is an independent coin flip, so
|
||||
// the probability of the whole loop passing by luck is 2^-N: at
|
||||
// this N the test is deterministic in practice, and it involves no
|
||||
// wall-clock waiting at all.
|
||||
const racePasses = 1000
|
||||
|
||||
// TestWaitDone_DrainedBeforeExpiredContext covers the case where a
|
||||
// component drained cleanly but the stop context had already
|
||||
// expired. Both select cases are ready, and Go chooses among ready
|
||||
// cases uniformly at random, so the drained case must be settled by
|
||||
// the preamble before the bounded select ever runs.
|
||||
func TestWaitDone_DrainedBeforeExpiredContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
done := make(chan struct{})
|
||||
close(done)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
for pass := range racePasses {
|
||||
require.NoErrorf(
|
||||
t,
|
||||
lifecycle.WaitDone(
|
||||
ctx, discardLogger(), "test component", done,
|
||||
),
|
||||
"pass %d reported a timeout for a drained component",
|
||||
pass,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWaitDone_ExpiredContext pins the other side of the preamble:
|
||||
// an expired context with a component that has not drained is still
|
||||
// a timeout.
|
||||
func TestWaitDone_ExpiredContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
err := lifecycle.WaitDone(
|
||||
ctx, discardLogger(), "test component",
|
||||
make(chan struct{}),
|
||||
)
|
||||
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.ErrorContains(t, err, "test component")
|
||||
}
|
||||
|
||||
func TestWaitForShutdown_ContextExpires(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -24,15 +24,48 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// shutdownTimeout is the maximum time to wait for the HTTP
|
||||
// ShutdownTimeout is the maximum time to wait for the HTTP
|
||||
// server to finish in-flight requests during shutdown.
|
||||
shutdownTimeout = 5 * time.Second
|
||||
//
|
||||
// It must stay strictly below the fx stop timeout in
|
||||
// cmd/webhooker, which bounds the whole stop sequence: a drain
|
||||
// that used the entire sequence budget would leave nothing for
|
||||
// the hooks that run after the server, including the database
|
||||
// close. It is exported so that relationship can be tested.
|
||||
ShutdownTimeout = 3 * time.Second
|
||||
|
||||
// sentryFlushTimeout is the maximum time to wait for Sentry
|
||||
// to flush pending events during shutdown.
|
||||
// TailHookReserve is the share of the fx stop budget this hook
|
||||
// refuses to spend, leaving it for the hooks that run after the
|
||||
// server: the delivery engine, the healthcheck, the webhook DB
|
||||
// manager and the database close.
|
||||
TailHookReserve = 2 * time.Second
|
||||
|
||||
// sentryFlushTimeout is the longest wait for Sentry to flush
|
||||
// pending events during shutdown, before the remaining stop
|
||||
// budget is taken into account.
|
||||
sentryFlushTimeout = 2 * time.Second
|
||||
|
||||
// minSentryFlush is the shortest flush worth attempting. Below
|
||||
// it the remaining budget goes to the tail hooks instead.
|
||||
minSentryFlush = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
// SentryFlushBudget reports how long the Sentry flush may run when
|
||||
// remaining is the time left on the fx stop context after the HTTP
|
||||
// drain. sentry.Flush takes a bare duration and honours no context,
|
||||
// so this clamp is the only thing keeping a stalled flush from
|
||||
// spending the tail hooks' share of the budget on top of a
|
||||
// full-length drain. TailHookReserve is held back, and anything
|
||||
// under minSentryFlush is skipped rather than attempted uselessly.
|
||||
func SentryFlushBudget(remaining time.Duration) time.Duration {
|
||||
budget := min(remaining-TailHookReserve, sentryFlushTimeout)
|
||||
if budget < minSentryFlush {
|
||||
return 0
|
||||
}
|
||||
|
||||
return budget
|
||||
}
|
||||
|
||||
//nolint:revive // ServerParams is a standard fx naming convention.
|
||||
type ServerParams struct {
|
||||
fx.In
|
||||
@@ -164,7 +197,7 @@ func (s *Server) cleanShutdown(ctx context.Context) {
|
||||
s.exitCode = 0
|
||||
|
||||
ctxShutdown, shutdownCancel := context.WithTimeout(
|
||||
ctx, shutdownTimeout,
|
||||
ctx, ShutdownTimeout,
|
||||
)
|
||||
defer shutdownCancel()
|
||||
|
||||
@@ -178,10 +211,31 @@ func (s *Server) cleanShutdown(ctx context.Context) {
|
||||
s.cleanupForExit()
|
||||
|
||||
if s.sentryEnabled {
|
||||
sentry.Flush(sentryFlushTimeout)
|
||||
s.flushSentry(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// flushSentry drains Sentry's queue inside what is left of the fx
|
||||
// stop budget. A context carrying no deadline — a caller outside the
|
||||
// fx lifecycle — gets the full timeout.
|
||||
func (s *Server) flushSentry(ctx context.Context) {
|
||||
flush := sentryFlushTimeout
|
||||
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
flush = SentryFlushBudget(time.Until(deadline))
|
||||
}
|
||||
|
||||
if flush <= 0 {
|
||||
s.log.Warn(
|
||||
"skipping sentry flush, stop budget exhausted",
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
sentry.Flush(flush)
|
||||
}
|
||||
|
||||
func (s *Server) configure() {
|
||||
// identify ourselves in the logs
|
||||
s.params.Logger.Identify()
|
||||
|
||||
59
internal/server/shutdown_test.go
Normal file
59
internal/server/shutdown_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package server_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/server"
|
||||
)
|
||||
|
||||
// TestSentryFlushBudget covers the clamp that keeps the Sentry flush
|
||||
// from spending the tail hooks' share of the fx stop budget.
|
||||
// sentry.Flush ignores the stop context, so without the clamp a
|
||||
// stalled flush adds its whole timeout on top of the HTTP drain.
|
||||
func TestSentryFlushBudget(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
remaining time.Duration
|
||||
want time.Duration
|
||||
}{
|
||||
{
|
||||
name: "full drain leaves only the reserve",
|
||||
remaining: server.TailHookReserve,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "expired budget",
|
||||
remaining: -time.Second,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "sliver above the reserve is not worth it",
|
||||
remaining: server.TailHookReserve + 10*time.Millisecond,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "partial flush when some room is left",
|
||||
remaining: server.TailHookReserve + time.Second,
|
||||
want: time.Second,
|
||||
},
|
||||
{
|
||||
name: "capped at the nominal timeout",
|
||||
remaining: time.Hour,
|
||||
want: 2 * time.Second,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
require.Equal(
|
||||
t, tt.want, server.SentryFlushBudget(tt.remaining),
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
152
script/ci-mark-superseded
Executable file
152
script/ci-mark-superseded
Executable file
@@ -0,0 +1,152 @@
|
||||
#!/bin/sh
|
||||
# script/ci-mark-superseded: record an honest status on commits whose CI
|
||||
# run Gitea cancelled because a newer commit landed on the same branch.
|
||||
# Gitea writes `failure` / "Has been cancelled" for such a run, which
|
||||
# reads as a test result on a commit nothing ever tested. Cancellation is
|
||||
# unconditional server-side for push events, so the superseding run
|
||||
# rewrites those statuses to `failure` with a description that says the
|
||||
# commit was never tested. `skipped` cannot be used: Gitea's combined
|
||||
# status folds `skipped` into `success`, so a never-tested commit would
|
||||
# report green. Genuine failures and successes are never touched.
|
||||
#
|
||||
# Called by the Gitea Actions workflow, which supplies GITHUB_API_URL,
|
||||
# GITHUB_REPOSITORY, GITHUB_SHA, GITHUB_WORKFLOW, GITHUB_JOB,
|
||||
# GITHUB_EVENT_NAME and GITEA_TOKEN. ANCESTOR_LIMIT (default 20) caps how
|
||||
# far back the walk looks; a value that is set but not a positive integer
|
||||
# aborts rather than silently disabling the walk.
|
||||
set -eu
|
||||
|
||||
SUPERSEDED_DESC='Superseded by a newer commit; never tested'
|
||||
|
||||
# Gitea builds the commit-status context as
|
||||
# "<workflow name> / <job name> (<event>)", so derive it rather than
|
||||
# hardcoding the result.
|
||||
#
|
||||
# The derivation is deliberately not byte-exact with Gitea's own rule and
|
||||
# must not be "fixed" into a silent fallback. Gitea uses the job's `name:`
|
||||
# (falling back to the job id) and the workflow's `name:` (falling back to
|
||||
# the workflow filename), while the runner exports GITHUB_JOB as the job
|
||||
# *id* and GITHUB_WORKFLOW as the parsed workflow `name:`. So giving the
|
||||
# job a display `name:`, or dropping the workflow's `name:`, makes the
|
||||
# derived context stop matching --- and require_own_context below then
|
||||
# turns every push red with a message. That loud failure is the point
|
||||
# (https://git.eeqj.de/sneak/webhooker/issues/147 item 2); guessing at a
|
||||
# fallback would restore the silent no-op it replaced.
|
||||
context() {
|
||||
printf '%s / %s (%s)' \
|
||||
"$GITHUB_WORKFLOW" "$GITHUB_JOB" "$GITHUB_EVENT_NAME"
|
||||
}
|
||||
|
||||
# ANCESTOR_LIMIT is a documented knob, so a value that is set but
|
||||
# unusable must fail loudly instead of defaulting
|
||||
# (https://git.eeqj.de/sneak/webhooker/issues/80). Passing it straight to
|
||||
# git would print `fatal: not an integer` into a discarded exit status
|
||||
# and mark nothing.
|
||||
ancestor_limit() {
|
||||
# `-` and not `:-`: an explicitly empty value is set-but-unusable
|
||||
# config, so it aborts like any other bad value rather than silently
|
||||
# running at the default.
|
||||
_limit="${ANCESTOR_LIMIT-20}"
|
||||
case "$_limit" in
|
||||
'' | *[!0-9]* | 0*)
|
||||
echo "ANCESTOR_LIMIT must be a positive integer," \
|
||||
"got '${_limit}'" >&2
|
||||
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
printf '%s' "$_limit"
|
||||
}
|
||||
|
||||
# The status Gitea created for this very job proves which context string
|
||||
# it uses. If the derived one is missing, the workflow or the job was
|
||||
# renamed and the match below would silently stop firing, restoring the
|
||||
# false-red bug with no signal. Fail loudly instead.
|
||||
require_own_context() {
|
||||
if ! _body="$(curl -sf --retry 3 --retry-delay 2 --max-time 30 \
|
||||
"${1}/commits/${GITHUB_SHA}/status")"; then
|
||||
echo "cannot read commit statuses for ${GITHUB_SHA}" >&2
|
||||
return 1
|
||||
fi
|
||||
_found="$(printf '%s' "$_body" | jq -r '(.statuses // [])[].context')"
|
||||
if printf '%s\n' "$_found" | grep -qxF "$2"; then
|
||||
return 0
|
||||
fi
|
||||
echo "no commit status with context '${2}' on ${GITHUB_SHA}:" >&2
|
||||
echo "workflow or job renamed? contexts present:" >&2
|
||||
printf '%s\n' "$_found" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# Latest status for our context on a commit, as "state|description".
|
||||
# The read is retried and bounded, and a read that still fails aborts the
|
||||
# step: a laundered commit that cannot be read is not the same as one
|
||||
# with nothing to do, and piping curl into jq would discard the
|
||||
# difference.
|
||||
status_of() {
|
||||
if ! _sbody="$(curl -sf --retry 3 --retry-delay 2 --max-time 30 \
|
||||
"${1}/commits/${2}/status")"; then
|
||||
echo "cannot read commit statuses for ${2}" >&2
|
||||
|
||||
return 1
|
||||
fi
|
||||
printf '%s' "$_sbody" | jq -r --arg c "$3" \
|
||||
'[(.statuses // [])[] | select(.context == $c)][0] // empty
|
||||
| "\(.status)|\(.description)"'
|
||||
}
|
||||
|
||||
mark_superseded() {
|
||||
curl -sf -X POST "${1}/statuses/${2}" \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "$(jq -nc --arg c "$3" --arg d "$SUPERSEDED_DESC" \
|
||||
'{context: $c, state: "failure", description: $d}')" \
|
||||
>/dev/null
|
||||
}
|
||||
|
||||
main() {
|
||||
_api="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}"
|
||||
_ctx="$(context)"
|
||||
|
||||
_limit="$(ancestor_limit)"
|
||||
|
||||
require_own_context "$_api" "$_ctx"
|
||||
|
||||
# A shallow clone cannot resolve the parent, so it looks exactly like
|
||||
# a root commit to rev-parse below and would exit 0 having walked
|
||||
# nothing (or, at depth > 1, only the ancestors that happen to be
|
||||
# present). The workflow checks out with `fetch-depth: 0`; verify
|
||||
# that here rather than depend on it silently.
|
||||
if [ "$(git rev-parse --is-shallow-repository)" = 'true' ]; then
|
||||
echo "shallow repository: the ancestor walk needs full history" >&2
|
||||
|
||||
return 1
|
||||
fi
|
||||
|
||||
# A root commit legitimately has no ancestors and is not an error.
|
||||
# A SHA this repository does not have lands here too, since its
|
||||
# parent is equally unresolvable, but require_own_context above has
|
||||
# already aborted on the 404 for it. The walk itself carries no
|
||||
# `|| true`, so a rev-list failure aborts.
|
||||
if ! git rev-parse -q --verify "${GITHUB_SHA}^" >/dev/null; then
|
||||
echo "no ancestor of ${GITHUB_SHA} to check"
|
||||
|
||||
return 0
|
||||
fi
|
||||
|
||||
_walk="$(git rev-list --max-count="$_limit" "${GITHUB_SHA}^")"
|
||||
|
||||
for _sha in $_walk; do
|
||||
_latest="$(status_of "$_api" "$_sha" "$_ctx")"
|
||||
# A run that was cancelled, or one an earlier revision of this
|
||||
# script laundered into `skipped`. Anything else stands.
|
||||
case "$_latest" in
|
||||
'failure|Has been cancelled' | "skipped|${SUPERSEDED_DESC}") ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
mark_superseded "$_api" "$_sha" "$_ctx"
|
||||
echo "marked superseded: ${_sha}"
|
||||
done
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user