dcf/script/assert-step-ran

#!/bin/sh
# script/assert-step-ran: fail unless a docker build actually executed a
# named step, proved by output the step's own tool wrote.
#
# Usage: script/assert-step-ran LOG STAGE STEP_REGEX EVIDENCE_REGEX WHAT
#
# LOG must be `docker build --progress=plain` output. The default tty
# renderer rewrites lines in place and keeps only the tail of a step's
# output, so the flag is load-bearing wherever this is used.
#
# It passes only if some step in LOG satisfies all three of:
#
#   1. its header is "#ID [STAGE n/m] CMD" with CMD matching STEP_REGEX,
#   2. one of the lines that step wrote matches EVIDENCE_REGEX,
#   3. BuildKit reported "#ID DONE".
#
# This observes the build that happened; it parses no Dockerfile. A
# stage renamed, moved, made unreachable, deleted, split by a whitespace
# byte BuildKit treats as a separator, or an instruction rewritten —
# none of it can make this pass a build in which the step did not run,
# because a step that did not run wrote no output, and a step served
# from cache writes none either ("#ID CACHED", no output lines).
#
# The stage name in a header is BuildKit's label for that vertex, not a
# reading of the file being built. A vertex shared with an earlier build
# of a DIFFERENT Dockerfile is replayed under the label it was first
# recorded with, so a `Dockerfile.lint` run can print `[lint 1/8]` for
# the main `Dockerfile`'s lint stage. A borrowed label cannot produce a
# pass — a replayed vertex prints CACHED and writes no evidence, and a
# vertex that really executes really ran the command in the header — but
# it does mean a header alone proves nothing about which stages the file
# declares. Hence one failure message naming both causes rather than a
# split that would claim to know which.
#
# What this guard cannot see — not an exhaustive list of ways a green can
# be untrue, only the ones known:
#
#   - Evidence forged inside the matched step, e.g.
#     `RUN golangci-lint run ... || echo "0 issues."`. Condition 1 ties
#     the evidence to a step whose own command is in the log, so the
#     forgery has to be written into that command in the Dockerfile, in
#     plain sight.
# This guards against a step falling silently out of the build. It does
# not certify that what ran examined everything it should have — a
# `.dockerignore` entry excluding a package makes the linter genuinely
# run and genuinely print `0 issues.` while a real violation sits
# unexamined in the repo. That is a different question, asked separately
# by script/assert-context-complete, which every caller of this script
# also runs.
#
# What it depends on — BuildKit's plain progress format and the tool's
# own success wording — fails the caller loudly if it drifts, because
# drift removes a match rather than creating one.
set -eu

die() {
    echo "script/assert-step-ran: $*" >&2
    exit 1
}

[ $# -eq 5 ] || die "usage: $0 LOG STAGE STEP_REGEX EVIDENCE_REGEX WHAT"
[ -r "$1" ] || die "cannot read $1"

# The regexes travel in the environment, not in -v: awk expands escape
# sequences in a -v assignment, which would eat a backslash before the
# regex ever sees it.
ASR_STAGE="$2" ASR_STEP="$3" ASR_EVIDENCE="$4" ASR_WHAT="$5" awk '
function fail(msg) {
    printf "script/assert-step-ran: %s\n", msg | "cat 1>&2"
    close("cat 1>&2")
    exit 1
}

BEGIN {
    stage = ENVIRON["ASR_STAGE"]
    step = ENVIRON["ASR_STEP"]
    evidence = ENVIRON["ASR_EVIDENCE"]
    what = ENVIRON["ASR_WHAT"]
}

# Every progress line is "#ID " and then a step header, a status, or one
# line the step itself wrote.
/^#[0-9]+ / {
    id = substr($1, 2)
    rest = substr($0, length($1) + 2)

    if (rest == "CACHED") { cached[id] = 1; next }
    if (rest ~ /^DONE /) { done[id] = 1; next }

    # Header: "[STAGE n/m] CMD", or "[PLATFORM STAGE n/m] CMD" when the
    # build names a platform. Bracketed spans with no n/m are BuildKit
    # internals ("[internal] load build context"), not steps.
    if (substr(rest, 1, 1) == "[") {
        p = index(rest, "] ")
        if (p == 0) next
        n = split(substr(rest, 2, p - 2), part, " ")
        if (n < 2 || part[n] !~ /^[0-9]+\/[0-9]+$/) next
        if (part[n - 1] != stage) next
        if (substr(rest, p + 2) ~ step) matched[id] = 1
        next
    }

    # Output: "#ID 41.80 <the line the step wrote>".
    if (rest ~ /^[0-9]+[.][0-9]+ /) {
        sub(/^[0-9]+[.][0-9]+ /, "", rest)
        if (rest ~ evidence) emitted[id] = 1
    }
}

END {
    for (id in matched) {
        nmatched++
        if (done[id] && emitted[id]) exit 0
        if (cached[id]) ncached++
    }

    if (nmatched == 0)
        fail(sprintf("the build ran no step matching /%s/ in stage \"%s\", so %s did not run. Either the stage is not in the build graph — renamed, deleted, or nothing the final stage builds depends on it any more — or it was built without that command among its steps", step, stage, what))

    if (ncached == nmatched)
        fail(sprintf("every step matching /%s/ in stage \"%s\" was served from cache, so %s did not run on this tree; the cache bust for that stage is not taking effect", step, stage, what))

    fail(sprintf("a step matching /%s/ ran in stage \"%s\" but never wrote a line matching /%s/, so there is no evidence %s did the work; the command or the tool that produces that line has changed", step, stage, evidence, what))
}
' "$1"