Mark superseded commits honestly instead of skipped (closes #152)

Gitea records a cancelled run as failure, and the #119 repair rewrote
that to skipped. Gitea's Combine() folds skipped into success, so a
commit nothing ever tested reported a combined green — observed on three
commits on next, including the very change a prior integration review
had failed a PR for.

Superseded commits are now marked failure with an honest description, so
never-tested no longer reads as passed and git bisect archaeology can
tell "passed", "failed" and "never ran" apart. Option 1, re-running the
superseded commit, was verified unreachable for automation on Gitea
1.25.4: no rerun endpoint, dispatches takes a ref not a SHA and lands
under a different context, and CancelPreviousJobs is unconditional.

The rewrite moves out of the workflow into script/ci-mark-superseded so
the tested artifact is the shipped one, and every failure path in it is
loud: an unparseable or empty ANCESTOR_LIMIT, an unreadable ancestor
status, and a shallow clone all abort rather than exiting 0 having
marked nothing. Each has a regression test. The status context is
derived rather than hardcoded, which also closes #147 item 2; item 1
remains open.

Independently reviewed four times. The final reviewer confirmed the
shallow-clone test is genuinely shallow — a file:// URL is load-bearing,
since git silently ignores --depth on a local path — and that deleting
the guard fails that one test out of 331 and cannot pass for the wrong
reason. They also reproduced deterministically that go test's cache
serves a stale PASS after a script-only edit, which internal/ciscript's
doc.go now records.

internal/ciscript/ci_mark_superseded_test.go

@@ -0,0 +1,387 @@
+package ciscript_test
+
+import (
+	"maps"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	// supersededDesc is the description script/ci-mark-superseded
+	// writes, and the one an earlier revision of it wrote alongside a
+	// `skipped` state.
+	supersededDesc = "Superseded by a newer commit; never tested"
+
+	// liveContext is the commit-status context Gitea uses for this
+	// repository's runs, as seen in its API. The script derives it from
+	// the workflow and job names rather than hardcoding it; the
+	// derivation is checked against this value below.
+	liveContext = "check / check (push)"
+
+	scriptPath = "../../script/ci-mark-superseded"
+	workflow   = "../../.gitea/workflows/check.yml"
+
+	// failure is the only state that neither folds into a combined
+	// `success` (as `skipped` does) nor blocks the commit forever (as
+	// `pending` does).
+	failure = "failure"
+)
+
+// repo is a throwaway git history: parent is the commit a run would be
+// cancelled on, head the commit that superseded it.
+type repo struct {
+	dir    string
+	head   string
+	parent string
+}
+
+// scriptEnv is the run identity the Gitea runner exports and the script
+// builds its context string from.
+type scriptEnv struct {
+	workflow string
+	job      string
+	event    string
+}
+
+func defaultEnv() scriptEnv {
+	return scriptEnv{workflow: "check", job: "check", event: "push"}
+}
+
+func cancelled() commitStatus {
+	return commitStatus{
+		Context:     liveContext,
+		Status:      failure,
+		Description: "Has been cancelled",
+	}
+}
+
+func running() commitStatus {
+	return commitStatus{
+		Context:     liveContext,
+		Status:      "pending",
+		Description: "Has started running",
+	}
+}
+
+func TestMarkSuperseded(t *testing.T) {
+	t.Parallel()
+
+	cases := map[string]struct {
+		parent   commitStatus
+		wantMark bool
+	}{
+		"a cancelled run is marked": {
+			parent:   cancelled(),
+			wantMark: true,
+		},
+		"a laundered skipped status is marked": {
+			parent: commitStatus{
+				Context:     liveContext,
+				Status:      "skipped",
+				Description: supersededDesc,
+			},
+			wantMark: true,
+		},
+		"a genuine failure is left alone": {
+			parent: commitStatus{
+				Context:     liveContext,
+				Status:      failure,
+				Description: "Failing after 3m1s",
+			},
+			wantMark: false,
+		},
+		"a passing run is left alone": {
+			parent: commitStatus{
+				Context:     liveContext,
+				Status:      "success",
+				Description: "Successful in 2m52s",
+			},
+			wantMark: false,
+		},
+		"another context is left alone": {
+			parent: commitStatus{
+				Context:     "other / other (push)",
+				Status:      failure,
+				Description: "Has been cancelled",
+			},
+			wantMark: false,
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			requireTools(t)
+
+			history := newRepo(t)
+			fake, api := newFakeGitea(t)
+			fake.setStatus(history.head, running())
+			fake.setStatus(history.parent, tc.parent)
+
+			out, err := runScript(t, history, api, defaultEnv())
+			require.NoError(t, err, out)
+
+			posted := fake.postedFor(history.parent)
+			if !tc.wantMark {
+				require.Empty(t, posted)
+
+				return
+			}
+
+			require.Equal(t, []postedStatus{{
+				Context: liveContext,
+				// Not `skipped`: Gitea's combined status folds
+				// that into `success`, which is what made a
+				// never-tested commit read green.
+				State:       failure,
+				Description: supersededDesc,
+			}}, posted)
+		})
+	}
+}
+
+// A second run must not rewrite what the first one wrote, or every
+// later push would post a duplicate status.
+func TestMarkSupersededIsIdempotent(t *testing.T) {
+	t.Parallel()
+	requireTools(t)
+
+	history := newRepo(t)
+	fake, api := newFakeGitea(t)
+	fake.setStatus(history.head, running())
+	fake.setStatus(history.parent, cancelled())
+
+	for range 2 {
+		out, err := runScript(t, history, api, defaultEnv())
+		require.NoError(t, err, out)
+	}
+
+	require.Len(t, fake.postedFor(history.parent), 1)
+}
+
+// Renaming the workflow or the job changes the context string Gitea
+// uses. The script must say so instead of quietly matching nothing.
+func TestMarkSupersededRejectsAnUnknownContext(t *testing.T) {
+	t.Parallel()
+	requireTools(t)
+
+	history := newRepo(t)
+	fake, api := newFakeGitea(t)
+	fake.setStatus(history.head, running())
+	fake.setStatus(history.parent, cancelled())
+
+	env := defaultEnv()
+	env.job = "renamed"
+
+	out, err := runScript(t, history, api, env)
+	require.Error(t, err)
+	require.Contains(t, out, "renamed")
+	require.Contains(t, out, liveContext)
+	require.Empty(t, fake.postedFor(history.parent))
+}
+
+// ANCESTOR_LIMIT is a documented knob. A value that is set but unusable
+// must abort: handing it to git and discarding the exit status left the
+// walk empty and the step green, marking nothing.
+func TestMarkSupersededRejectsAnUnparseableAncestorLimit(t *testing.T) {
+	t.Parallel()
+	requireTools(t)
+
+	history := newRepo(t)
+	fake, api := newFakeGitea(t)
+	fake.setStatus(history.head, running())
+	fake.setStatus(history.parent, cancelled())
+
+	out, err := runScript(
+		t, history, api, defaultEnv(), "ANCESTOR_LIMIT=twenty",
+	)
+	require.Error(t, err)
+	require.Contains(t, out, "ANCESTOR_LIMIT")
+	require.Contains(t, out, "twenty")
+	require.Empty(t, fake.postedFor(history.parent))
+}
+
+// A status read that fails is not the same as a commit with nothing to
+// do. Losing curl's exit status through a pipe made the two identical
+// and left a laundered commit laundered with no signal.
+func TestMarkSupersededFailsOnAnUnreadableAncestorStatus(t *testing.T) {
+	t.Parallel()
+	requireTools(t)
+
+	history := newRepo(t)
+	fake, api := newFakeGitea(t)
+	fake.setStatus(history.head, running())
+	fake.setStatus(history.parent, cancelled())
+	fake.failStatusRead(history.parent)
+
+	out, err := runScript(t, history, api, defaultEnv())
+	require.Error(t, err)
+	require.Contains(t, out, history.parent)
+	require.Contains(t, out, "cannot read commit statuses")
+	require.Empty(t, fake.postedFor(history.parent))
+}
+
+// A shallow clone cannot resolve the parent, so it is indistinguishable
+// from a root commit to rev-parse and the walk would exit 0 having
+// marked nothing. It must abort instead: dropping `fetch-depth: 0` from
+// the checkout step is one edit, and a silent no-op there restores the
+// false-green bug this script exists to prevent.
+func TestMarkSupersededRejectsAShallowRepository(t *testing.T) {
+	t.Parallel()
+	requireTools(t)
+
+	history := shallowClone(t, newRepo(t))
+	fake, api := newFakeGitea(t)
+	fake.setStatus(history.head, running())
+	fake.setStatus(history.parent, cancelled())
+
+	out, err := runScript(t, history, api, defaultEnv())
+	require.Error(t, err)
+	require.Contains(t, out, "shallow repository")
+	require.Empty(t, fake.postedFor(history.parent))
+	require.Empty(t, fake.postedFor(history.head))
+}
+
+// shallowClone returns the same history as a depth-1 clone. The `file://`
+// URL is required: git ignores --depth for a plain local path.
+func shallowClone(t *testing.T, history repo) repo {
+	t.Helper()
+
+	dir := t.TempDir()
+
+	//nolint:gosec // fixed argv, arguments are test-local paths
+	cmd := exec.CommandContext(t.Context(), "git", "clone", "-q",
+		"--depth=1", "file://"+history.dir, dir)
+
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(out))
+
+	return repo{dir: dir, head: history.head, parent: history.parent}
+}
+
+// The derived context must equal the one Gitea actually uses, which is
+// built from the same workflow and job names.
+func TestDerivedContextMatchesGitea(t *testing.T) {
+	t.Parallel()
+	requireTools(t)
+
+	name, job := workflowIdentity(t)
+
+	history := newRepo(t)
+	fake, api := newFakeGitea(t)
+	fake.setStatus(history.head, running())
+	fake.setStatus(history.parent, cancelled())
+
+	out, err := runScript(t, history, api, scriptEnv{
+		workflow: name,
+		job:      job,
+		event:    "push",
+	})
+	require.NoError(t, err, out)
+
+	posted := fake.postedFor(history.parent)
+	require.Len(t, posted, 1)
+	require.Equal(t, liveContext, posted[0].Context)
+}
+
+// workflowIdentity reads the workflow name and its single job id out of
+// the checked-in workflow file.
+func workflowIdentity(t *testing.T) (string, string) {
+	t.Helper()
+
+	raw, err := os.ReadFile(workflow)
+	require.NoError(t, err)
+
+	var parsed struct {
+		Name string         `yaml:"name"`
+		Jobs map[string]any `yaml:"jobs"`
+	}
+
+	require.NoError(t, yaml.Unmarshal(raw, &parsed))
+
+	jobs := slices.Collect(maps.Keys(parsed.Jobs))
+	require.Len(t, jobs, 1)
+
+	return parsed.Name, jobs[0]
+}
+
+func runScript(
+	t *testing.T, history repo, api string, env scriptEnv,
+	extra ...string,
+) (string, error) {
+	t.Helper()
+
+	script, err := filepath.Abs(scriptPath)
+	require.NoError(t, err)
+
+	//nolint:gosec // fixed argv, repo-local script under test
+	cmd := exec.CommandContext(t.Context(), "sh", script)
+	cmd.Dir = history.dir
+	cmd.Env = append(os.Environ(),
+		"GITHUB_API_URL="+api,
+		"GITHUB_REPOSITORY=sneak/webhooker",
+		"GITHUB_SHA="+history.head,
+		"GITHUB_WORKFLOW="+env.workflow,
+		"GITHUB_JOB="+env.job,
+		"GITHUB_EVENT_NAME="+env.event,
+		"GITEA_TOKEN=test-token",
+	)
+	cmd.Env = append(cmd.Env, extra...)
+
+	out, err := cmd.CombinedOutput()
+
+	return string(out), err
+}
+
+func newRepo(t *testing.T) repo {
+	t.Helper()
+
+	dir := t.TempDir()
+
+	git := func(args ...string) string {
+		//nolint:gosec // fixed argv, arguments are test constants
+		cmd := exec.CommandContext(t.Context(), "git", args...)
+		cmd.Dir = dir
+
+		out, err := cmd.CombinedOutput()
+		require.NoError(t, err, string(out))
+
+		return strings.TrimSpace(string(out))
+	}
+
+	commit := func(message string) string {
+		git(
+			"-c", "user.email=ci@example.invalid",
+			"-c", "user.name=ci",
+			"-c", "commit.gpgsign=false",
+			"commit", "-q", "--allow-empty", "-m", message,
+		)
+
+		return git("rev-parse", "HEAD")
+	}
+
+	git("init", "-q", "-b", "main")
+
+	parent := commit("parent")
+	head := commit("head")
+
+	return repo{dir: dir, head: head, parent: parent}
+}
+
+func requireTools(t *testing.T) {
+	t.Helper()
+
+	for _, tool := range []string{"sh", "git", "curl", "jq"} {
+		_, err := exec.LookPath(tool)
+		if err != nil {
+			t.Skipf("%s is not installed: %v", tool, err)
+		}
+	}
+}

internal/ciscript/doc.go

@@ -0,0 +1,10 @@
+// Package ciscript holds the tests for the repository's CI shell
+// scripts in script/. It carries no runtime code: the scripts run on
+// the CI runner, not inside the binary, but their behaviour still has
+// to be verified by the test suite.
+//
+// The scripts under test are outside the Go build graph, so `go test`'s
+// result cache serves a stale PASS when only a script changed: run the
+// container build, or GOFLAGS=-count=1, to trust a result here after
+// editing script/.
+package ciscript

internal/ciscript/fakegitea_test.go

@@ -0,0 +1,162 @@
+package ciscript_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+)
+
+// commitStatus is the part of an entry in Gitea's combined-status
+// response that script/ci-mark-superseded reads.
+type commitStatus struct {
+	Context     string `json:"context"`
+	Status      string `json:"status"`
+	Description string `json:"description"`
+}
+
+// postedStatus is the part of a create-status request body the script
+// writes.
+type postedStatus struct {
+	Context     string `json:"context"`
+	State       string `json:"state"`
+	Description string `json:"description"`
+}
+
+// fakeGitea serves the two endpoints the script talks to. Like Gitea,
+// the newest status for a context replaces the previous one, so a
+// second run of the script sees what the first one wrote.
+type fakeGitea struct {
+	mu       sync.Mutex
+	statuses map[string][]commitStatus
+	posted   map[string][]postedStatus
+	// failRead is a commit whose combined-status read answers HTTP
+	// 500, standing in for a status API that is down.
+	failRead string
+}
+
+// newFakeGitea returns the fake and the base URL to hand the script as
+// GITHUB_API_URL.
+func newFakeGitea(t *testing.T) (*fakeGitea, string) {
+	t.Helper()
+
+	fake := &fakeGitea{
+		mu:       sync.Mutex{},
+		statuses: map[string][]commitStatus{},
+		posted:   map[string][]postedStatus{},
+		failRead: "",
+	}
+
+	srv := httptest.NewServer(fake.routes())
+	t.Cleanup(srv.Close)
+
+	return fake, srv.URL
+}
+
+func (f *fakeGitea) routes() http.Handler {
+	mux := http.NewServeMux()
+
+	mux.HandleFunc(
+		"GET /repos/{owner}/{repo}/commits/{sha}/status",
+		f.handleCombined,
+	)
+	mux.HandleFunc(
+		"POST /repos/{owner}/{repo}/statuses/{sha}",
+		f.handleCreate,
+	)
+
+	return mux
+}
+
+func (f *fakeGitea) handleCombined(
+	w http.ResponseWriter, r *http.Request,
+) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	sha := r.PathValue("sha")
+	if f.failRead != "" && f.failRead == sha {
+		http.Error(w, "boom", http.StatusInternalServerError)
+
+		return
+	}
+
+	body := struct {
+		Statuses []commitStatus `json:"statuses"`
+	}{Statuses: f.statuses[sha]}
+
+	payload, err := json.Marshal(body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	_, _ = w.Write(payload)
+}
+
+func (f *fakeGitea) handleCreate(w http.ResponseWriter, r *http.Request) {
+	var got postedStatus
+
+	err := json.NewDecoder(r.Body).Decode(&got)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+
+		return
+	}
+
+	sha := r.PathValue("sha")
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	f.posted[sha] = append(f.posted[sha], got)
+	f.replaceLocked(sha, commitStatus{
+		Context:     got.Context,
+		Status:      got.State,
+		Description: got.Description,
+	})
+
+	w.WriteHeader(http.StatusCreated)
+}
+
+// failStatusRead makes the combined-status read for one commit answer
+// HTTP 500.
+func (f *fakeGitea) failStatusRead(sha string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	f.failRead = sha
+}
+
+// setStatus gives a commit its latest status for a context.
+func (f *fakeGitea) setStatus(sha string, status commitStatus) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	f.replaceLocked(sha, status)
+}
+
+// postedFor returns the statuses the script created for a commit.
+func (f *fakeGitea) postedFor(sha string) []postedStatus {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	return append([]postedStatus(nil), f.posted[sha]...)
+}
+
+// replaceLocked requires f.mu.
+func (f *fakeGitea) replaceLocked(sha string, status commitStatus) {
+	for i, existing := range f.statuses[sha] {
+		if existing.Context == status.Context {
+			f.statuses[sha][i] = status
+
+			return
+		}
+	}
+
+	f.statuses[sha] = append(f.statuses[sha], status)
+}