From 454f3410a11f9315970626d71fa980de84b76ff9 Mon Sep 17 00:00:00 2001 From: clawbot Date: Mon, 9 Nov 2026 18:32:46 +0000 Subject: [PATCH] Add script/install; script/update-policy runs it and commits (closes #165) script/install fetches the policy files from prompts over HTTP and writes them into a repo, installing itself and script/update-policy there. script/update-policy fetches the current script/install, runs it on its own repo and commits without pushing. The file list and the rules for keeping a repo's own entries move to script/install. Model: opus-5-5 --- Makefile | 5 +- README.md | 9 +- TODO.md | 5 + prompts/EXISTING_REPO_CHECKLIST.md | 14 +- prompts/NEW_REPO_CHECKLIST.md | 7 +- prompts/REPO_POLICIES.md | 36 ++- script/install | 246 ++++++++++++++++++ script/update-policy | 211 ++------------- test/update-policy.sh | 400 ++++++++++++++++++++--------- 9 files changed, 598 insertions(+), 335 deletions(-) create mode 100755 script/install diff --git a/Makefile b/Makefile index 6488473..c22ff51 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks update-policy +.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks install update-policy # Makefile targets are thin shims; the implementations live in script/ # per the scripts-to-rule-them-all pattern (see the Entrypoints section @@ -31,5 +31,8 @@ docker: hooks: @script/install-precommit +install: + @script/install + update-policy: @script/update-policy diff --git a/README.md b/README.md index 740c99a..e35efd5 100644 --- a/README.md +++ b/README.md @@ -141,9 +141,12 @@ alpine. We provide: `script/check` - `script/install-precommit` — installs the git pre-commit hook (our own extension); `make hooks` shims to it -- `script/update-policy` — fetch the vendored policy files from this repo, - commit them and push (our own extension); it is for the repos that vendor them - and stops when run here +- `script/install` — fetch the vendored policy files from this repo over HTTP + and write them into the repo given as its argument, or its own repo with no + argument (our own extension); it stops when run here +- `script/update-policy` — fetch the current `script/install`, run it on its own + repo and commit the result without pushing (our own extension); it is for the + repos that vendor the policy files and stops when run here `make hooks` installs the pre-commit hook that runs `script/precommit`. diff --git a/TODO.md b/TODO.md index 3eb1e5b..31800dc 100644 --- a/TODO.md +++ b/TODO.md @@ -21,6 +21,11 @@ fmt-check, and commit. # Completed Steps +- 2026-11-09: `script/install` writes the vendored policy files into a repo over + HTTP, and installs itself and `script/update-policy` there + (https://git.eeqj.de/sneak/prompts/issues/165). `make update-policy` now + fetches the current `script/install`, runs it and commits without pushing. The + template repositories get both scripts in their own issues. - 2026-11-09: `script/update-policy` and `make update-policy` update a repo's vendored policy files with one command (https://git.eeqj.de/sneak/prompts/issues/162). They fetch the files from diff --git a/prompts/EXISTING_REPO_CHECKLIST.md b/prompts/EXISTING_REPO_CHECKLIST.md index 8c3849d..286cfcf 100644 --- a/prompts/EXISTING_REPO_CHECKLIST.md +++ b/prompts/EXISTING_REPO_CHECKLIST.md @@ -205,17 +205,19 @@ with your task. - [ ] `Makefile` exists in root — reference `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile` - [ ] Has targets: `test`, `lint`, `fmt`, `fmt-check`, `check`, `docker`, - `hooks`, `update-policy` + `hooks`, `install`, `update-policy` - [ ] Target implementations live in `script/` (scripts-to-rule-them-all); Makefile targets are thin shims calling them — model scripts at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/` - [ ] `script/precommit` exists and the pre-commit hook (installed by `script/install-precommit`, shimmed by `make hooks`) runs it -- [ ] `script/update-policy` exists, identical to the model script, and - `make update-policy` shims to it. Before its first use, the repo's own - entries in `.gitignore`, `.editorconfig` and `.dockerignore` go below the - comment that ends each canonical file. An entry above that comment which - the canonical file has never held makes the script stop before it writes +- [ ] `script/install` and `script/update-policy` exist, identical to the model + scripts, and `make install` and `make update-policy` shim to them. + `make update-policy` commits without pushing, so a person reviews and + pushes the commit. Before its first use, the repo's own entries in + `.gitignore`, `.editorconfig` and `.dockerignore` go below the comment + that ends each canonical file. An entry above that comment which the + canonical file has never held makes the script stop before it writes anything, naming the file and the entry. It stops the same way on a line in `.golangci.yml` that the canonical file has never held, apart from comments and the repo's entries in the `deny` list of the `test-support` diff --git a/prompts/NEW_REPO_CHECKLIST.md b/prompts/NEW_REPO_CHECKLIST.md index ff769b1..cc00df2 100644 --- a/prompts/NEW_REPO_CHECKLIST.md +++ b/prompts/NEW_REPO_CHECKLIST.md @@ -266,9 +266,12 @@ are thin shims calling them. Model scripts: - [ ] `script/install-precommit` — installs the pre-commit hook that runs `script/precommit` - [ ] `make hooks` — shims to `script/install-precommit` -- [ ] `script/update-policy` / `make update-policy` — fetches the vendored - policy files from the `prompts` repo, commits them and pushes; identical +- [ ] `script/install` / `make install` — fetches the vendored policy files from + the `prompts` repo over HTTP and writes them into the repo; identical across repos +- [ ] `script/update-policy` / `make update-policy` — fetches the current + `script/install`, runs it on the repo and commits the result without + pushing; identical across repos - [ ] README **Entrypoints** section documents the scripts and links the standard diff --git a/prompts/REPO_POLICIES.md b/prompts/REPO_POLICIES.md index 528bcfa..0a1a783 100644 --- a/prompts/REPO_POLICIES.md +++ b/prompts/REPO_POLICIES.md @@ -93,8 +93,8 @@ style conventions are in separate documents: - `make check` (runs `test`, `lint`, `fmt-check`) - `make docker` - `make hooks` (installs pre-commit hook) - - `make update-policy` (fetches the vendored policy files, commits and - pushes) + - `make install` (writes the current policy files into the repo) + - `make update-policy` (brings the repo up to standards and commits) A model Makefile is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`. @@ -111,6 +111,7 @@ style conventions are in separate documents: - `script/fmt-check` - `script/check` - `script/docker` + - `script/install` - `script/update-policy` The scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they @@ -185,15 +186,18 @@ style conventions are in separate documents: host. On a pristine checkout with nothing installed, the run dies there, after the containerised gates have passed. - Five further scripts are our own extensions to the standard: + Six further scripts are our own extensions to the standard: - `script/check` runs `script/test`, `script/lint` and `script/fmt-check`. - `script/precommit` is what the git pre-commit hook runs, and it calls `script/check`. - `script/install-precommit` installs the git pre-commit hook. The `make hooks` target shims to it. - - `script/update-policy` fetches the vendored policy files from the - `prompts` repo, commits them and pushes. The `make update-policy` target - shims to it. + - `script/install` fetches the vendored policy files from the `prompts` repo + over HTTP and writes them into the repo it is given, or its own repo with + no argument. The `make install` target shims to it. + - `script/update-policy` fetches the current `script/install` from the + `prompts` repo, runs it on its own repo and commits the result. It does + not push. The `make update-policy` target shims to it. - `script/projectname` (literally that filename) simply outputs the project's name. @@ -933,12 +937,18 @@ style conventions are in separate documents: - Make all changes on a feature branch. You can do whatever you want on a feature branch. -- A repo updates its policy files with one command: `make update-policy`. It - runs `script/update-policy`, which fetches each vendored file the repo carries - from the `prompts` repo, commits them and pushes. It uses the head of `main`, - or the commit given by a full hash: `script/update-policy `. The list of - vendored files is at the top of that script. The script replaces each file and - keeps only the repo's own entries: +- One command brings a repo up to standards: `make update-policy`. It runs + `script/update-policy`, which fetches the current `script/install` from the + head of `main` in the `prompts` repo and runs it on the repo. It then commits + the files that changed and does not push, so a person reviews and pushes the + commit. + + `script/install` fetches each vendored file the repo carries from `prompts` + over HTTP. It also installs `script/install` and `script/update-policy` in + the repo, so a repo updated once can update itself. Every repo made from a + template repo already has both. The list of vendored files is at the top of + `script/install`. The script replaces each file and keeps only the repo's + own entries: - in `.gitignore`, `.editorconfig` and `.dockerignore`: the lines below the comment that ends the canonical file - in `.golangci.yml`: the entries the repo added to the `deny` list of the @@ -1169,7 +1179,7 @@ style conventions are in separate documents: - `Makefile` - `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, - `install-precommit`, `update-policy`) + `install-precommit`, `install`, `update-policy`) - `Dockerfile`, `.dockerignore` - `.gitea/workflows/check.yml` - Go: `go.mod`, `go.sum`, `.golangci.yml` diff --git a/script/install b/script/install new file mode 100755 index 0000000..788219f --- /dev/null +++ b/script/install @@ -0,0 +1,246 @@ +#!/bin/sh +# script/install: write the current policy files from the prompts repo +# into a repo clone, over HTTP. It commits nothing; script/update-policy +# does that. Identical in all repos. Our own extension to +# scripts-to-rule-them-all. +# +# Usage: script/install [path of a repo clone] +# Without a path it updates the repo this script sits in. +# +# Standard output is the full hash of the prompts commit, then each path +# written, one per line. Messages go to standard error. +# +# The script writes nothing until every file has been fetched and +# merged. A repo's own entries are kept: +# - .gitignore, .editorconfig, .dockerignore: the lines below the +# canonical file's last line. An entry above that line which the +# canonical file has never held stops the script. +# - .golangci.yml: the entries the repo added to the deny list of the +# test-support rule. Any other line the canonical file has never +# held stops the script, unless it is blank or a comment. +set -eu + +PROMPTS_URL="https://git.eeqj.de/sneak/prompts" +PROMPTS_API_URL="https://git.eeqj.de/api/v1/repos/sneak/prompts" +PROMPTS_FIRST_COMMIT="af257bef63f162a681bddc80d9b941c47a44b748" + +# Paths in the prompts repo. A file the target repo does not carry is +# skipped, except the two scripts, which every repo gets. +VENDORED_FILES=" +.dockerignore +.editorconfig +.gitea/workflows/check.yml +.gitignore +.golangci.yml +.prettierrc +prompts/API_POLICIES.md +prompts/CODE_STYLEGUIDE.md +prompts/CODE_STYLEGUIDE_GO.md +prompts/CODE_STYLEGUIDE_JS.md +prompts/CODE_STYLEGUIDE_PYTHON.md +prompts/EXISTING_REPO_CHECKLIST.md +prompts/GO_HTTP_SERVER_CONVENTIONS.md +prompts/LLM_PROSE_TELLS.md +prompts/NEW_REPO_CHECKLIST.md +prompts/REPO_POLICIES.md +script/install +script/update-policy +" + +die() { + echo "install: $*" >&2 + exit 1 +} + +# fetch +fetch() { + curl --fail --silent --show-error --max-time 60 --output "$2" "$1" || + die "cannot fetch $1" +} + +# local_path : print where the target repo keeps the +# file, or nothing if git does not track it there. A prompts/NAME.md +# lands at docs/NAME.md or NAME.md. +local_path() { + name="${1#prompts/}" + if [ "$name" != "$1" ] && [ -n "$(git ls-files -- "docs/$name")" ]; then + echo "docs/$name" + elif [ -n "$(git ls-files -- "$name")" ]; then + echo "$name" + fi +} + +# former_lines : print the lines earlier commits +# removed from the file. The file's section of the patch ends at the +# next diff or the next commit. +former_lines() { + awk -v header="diff --git a/$2 b/$2" ' + /^diff --git / { inside = ($0 == header); next } + /^From / { inside = 0; next } + inside && /^-/ { print substr($0, 2) } + ' "$1" +} + +# local_only_lines [end]: print each line of the local +# file, above line end if given, that the canonical file has never held: +# not in the fetched file or in .former, the lines earlier +# commits removed from it. Blank lines and comments do not count. +local_only_lines() { + cat "$1" "$1.former" | awk -v end="${3:-}" ' + NR == FNR { held[$0]; next } + (end == "" || FNR < end) && !/^[ \t]*(#|$)/ && !($0 in held) + ' - "$2" +} + +# keep_lines_below_last : append to the fetched file +# whatever follows its last line in the local file. Stop if the canonical +# file has never held a line above that one, unless it is blank or a +# comment: replacing the file would delete it. +keep_lines_below_last() { + last="$(tail -n 1 "$1")" + number="$(grep -n -F -x -- "$last" "$2" | head -n 1 | cut -d: -f1)" + if [ -z "$last" ] || [ -z "$number" ]; then + die "$2 lacks this line; add it and put the repo's own entries below it: $last" + fi + own="$(local_only_lines "$1" "$2" "$number")" + if [ -n "$own" ]; then + die "$2 has entries above its last canonical line that the canonical file lacks; move each below that line to keep it, or delete it: +$own" + fi + tail -n "+$((number + 1))" "$2" >> "$1" +} + +# keep_deny_entries : add to the deny list of the +# test-support rule in the fetched file the entries only the local file +# has. An entry is a "- pkg:" line, the lines under it and any comment +# above it. Stop if the local file still has a line that neither the +# result nor the canonical file has ever held, unless it is blank or a +# comment: replacing the file would delete it. +keep_deny_entries() { + awk ' + function indent(line) { return match(line, /[^ ]/) - 1 } + function print_local_entries(i) { + for (i = 1; i <= count; i++) + if (!(pkg[i] in fetched_pkg)) + printf "%s", entry[i] + } + + FNR == 1 { in_rule = 0; in_list = 0 } + + in_list && /[^ ]/ && indent($0) <= list_indent { + in_list = 0 + if (FILENAME == ARGV[2]) print_local_entries() + } + + in_list { + is_entry = ($1 == "-" && $2 == "pkg:") + if (FILENAME == ARGV[2]) { + if (is_entry) fetched_pkg[$3] + print + } else if (is_entry) { + count++ + pkg[count] = $3 + entry[count] = comment $0 "\n" + comment = "" + } else if (/^ *#/) { + comment = comment $0 "\n" + } else if (count) { + entry[count] = entry[count] $0 "\n" + } + next + } + + in_rule && /^ *[^ #]/ && indent($0) <= rule_indent { in_rule = 0 } + /^ *test-support:/ { in_rule = 1; rule_indent = indent($0) } + in_rule && /^ *deny:/ { + in_rule = 0 + in_list = 1 + list_indent = indent($0) + if (FILENAME == ARGV[2]) found = 1 + } + FILENAME == ARGV[2] { print } + + END { + if (in_list) print_local_entries() + exit !found + } + ' "$2" "$1" > "$1.merged" || + die "the fetched $2 has no deny list in its test-support rule" + mv "$1.merged" "$1" + own="$(local_only_lines "$1" "$2")" + if [ -n "$own" ]; then + die "$2 has lines the canonical file has never held; only the deny list of the test-support rule may differ from canonical: +$own" + fi +} + +main() { + [ "$#" -le 1 ] || die "usage: script/install [path of a repo clone]" + + target="${1:-$(dirname "$0")/..}" + cd "$target" || die "no such directory: $target" + if [ "$(git rev-parse --show-toplevel 2> /dev/null)" != "$(pwd -P)" ]; then + die "not the top of a git clone: $target" + fi + if [ -e prompts/REPO_POLICIES.md ]; then + die "this repo is the source of the policy files" + fi + + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' EXIT + + fetch "$PROMPTS_API_URL/branches/main" "$tmp/branch.json" + commit="$(grep -Eo '"id":"[0-9a-f]{40}"' "$tmp/branch.json" | + head -n 1 | cut -d'"' -f4)" + [ -n "$commit" ] || die "no commit hash in $PROMPTS_API_URL/branches/main" + + fetch "$PROMPTS_URL/compare/$PROMPTS_FIRST_COMMIT...$commit.patch" \ + "$tmp/history.patch" + + carried="" + for file in $VENDORED_FILES; do + case "$file" in + script/*) path="$file" ;; + *) path="$(local_path "$file")" ;; + esac + [ -n "$path" ] || continue + if [ -n "$(git status --porcelain -- "$path")" ]; then + die "$path has uncommitted changes" + fi + mkdir -p "$tmp/$(dirname "$path")" + fetch "$PROMPTS_URL/raw/commit/$commit/$file" "$tmp/$path" + former_lines "$tmp/history.patch" "$file" > "$tmp/$path.former" + carried="$carried $path" + done + + for path in $carried; do + case "$path" in + .gitignore | .editorconfig | .dockerignore) + keep_lines_below_last "$tmp/$path" "$path" + ;; + .golangci.yml) + keep_deny_entries "$tmp/$path" "$path" + ;; + esac + done + + echo "$commit" + for path in $carried; do + case "$path" in + script/*) + # A new name and mv, because script/update-policy may + # be the script running. + mkdir -p script + cp "$tmp/$path" "$path.new" + chmod 755 "$path.new" + mv "$path.new" "$path" + ;; + *) + cp "$tmp/$path" "$path" + ;; + esac + echo "$path" + done +} + +main "$@" diff --git a/script/update-policy b/script/update-policy index eaf4bea..d58aa88 100755 --- a/script/update-policy +++ b/script/update-policy @@ -1,222 +1,43 @@ #!/bin/sh -# script/update-policy: replace this repo's vendored policy files with -# the canonical ones from the prompts repo, commit them and push. -# Identical in all repos. Our own extension to -# scripts-to-rule-them-all. -# -# Usage: script/update-policy [full commit hash of prompts] -# Without a hash it uses the head of the prompts main branch. -# -# The script writes nothing until every file has been fetched and -# merged. A repo's own entries are kept: -# - .gitignore, .editorconfig, .dockerignore: the lines below the -# canonical file's last line. An entry above that line which the -# canonical file has never held stops the script. -# - .golangci.yml: the entries the repo added to the deny list of the -# test-support rule. Any other line the canonical file has never -# held stops the script, unless it is blank or a comment. +# script/update-policy: bring this repo up to standards. It fetches the +# current script/install from the prompts repo over HTTP, runs it on this +# repo and commits the files it wrote. It does not push. Identical in all +# repos. Our own extension to scripts-to-rule-them-all. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" -PROMPTS_URL="https://git.eeqj.de/sneak/prompts" - -# Paths in the prompts repo. A file this repo does not carry is skipped. -VENDORED_FILES=" -.dockerignore -.editorconfig -.gitea/workflows/check.yml -.gitignore -.golangci.yml -.prettierrc -prompts/API_POLICIES.md -prompts/CODE_STYLEGUIDE.md -prompts/CODE_STYLEGUIDE_GO.md -prompts/CODE_STYLEGUIDE_JS.md -prompts/CODE_STYLEGUIDE_PYTHON.md -prompts/EXISTING_REPO_CHECKLIST.md -prompts/GO_HTTP_SERVER_CONVENTIONS.md -prompts/LLM_PROSE_TELLS.md -prompts/NEW_REPO_CHECKLIST.md -prompts/REPO_POLICIES.md -" +INSTALL_URL="https://git.eeqj.de/sneak/prompts/raw/branch/main/script/install" die() { echo "update-policy: $*" >&2 exit 1 } -# local_path : print where this repo keeps the file, or -# nothing if git does not track it here. A prompts/NAME.md lands at -# docs/NAME.md or NAME.md. -local_path() { - name="${1#prompts/}" - if [ "$name" != "$1" ] && [ -n "$(git ls-files -- "docs/$name")" ]; then - echo "docs/$name" - elif [ -n "$(git ls-files -- "$name")" ]; then - echo "$name" - fi -} - -# local_only_lines [end]: print each line of the local -# file, above line end if given, that the canonical file has never held: -# not in the fetched file or in .former, the lines earlier -# commits removed from it. Blank lines and comments do not count. -local_only_lines() { - cat "$1" "$1.former" | awk -v end="${3:-}" ' - NR == FNR { held[$0]; next } - (end == "" || FNR < end) && !/^[ \t]*(#|$)/ && !($0 in held) - ' - "$2" -} - -# keep_lines_below_last : append to the fetched file -# whatever follows its last line in the local file. Stop if the canonical -# file has never held a line above that one, unless it is blank or a -# comment: replacing the file would delete it. -keep_lines_below_last() { - last="$(tail -n 1 "$1")" - number="$(grep -n -F -x -- "$last" "$2" | head -n 1 | cut -d: -f1)" - if [ -z "$last" ] || [ -z "$number" ]; then - die "$2 lacks this line; add it and put the repo's own entries below it: $last" - fi - own="$(local_only_lines "$1" "$2" "$number")" - if [ -n "$own" ]; then - die "$2 has entries above its last canonical line that the canonical file lacks; move each below that line to keep it, or delete it: -$own" - fi - tail -n "+$((number + 1))" "$2" >> "$1" -} - -# keep_deny_entries : add to the deny list of the -# test-support rule in the fetched file the entries only the local file -# has. An entry is a "- pkg:" line, the lines under it and any comment -# above it. Stop if the local file still has a line that neither the -# result nor the canonical file has ever held, unless it is blank or a -# comment: replacing the file would delete it. -keep_deny_entries() { - awk ' - function indent(line) { return match(line, /[^ ]/) - 1 } - function print_local_entries(i) { - for (i = 1; i <= count; i++) - if (!(pkg[i] in fetched_pkg)) - printf "%s", entry[i] - } - - FNR == 1 { in_rule = 0; in_list = 0 } - - in_list && /[^ ]/ && indent($0) <= list_indent { - in_list = 0 - if (FILENAME == ARGV[2]) print_local_entries() - } - - in_list { - is_entry = ($1 == "-" && $2 == "pkg:") - if (FILENAME == ARGV[2]) { - if (is_entry) fetched_pkg[$3] - print - } else if (is_entry) { - count++ - pkg[count] = $3 - entry[count] = comment $0 "\n" - comment = "" - } else if (/^ *#/) { - comment = comment $0 "\n" - } else if (count) { - entry[count] = entry[count] $0 "\n" - } - next - } - - in_rule && /^ *[^ #]/ && indent($0) <= rule_indent { in_rule = 0 } - /^ *test-support:/ { in_rule = 1; rule_indent = indent($0) } - in_rule && /^ *deny:/ { - in_rule = 0 - in_list = 1 - list_indent = indent($0) - if (FILENAME == ARGV[2]) found = 1 - } - FILENAME == ARGV[2] { print } - - END { - if (in_list) print_local_entries() - exit !found - } - ' "$2" "$1" > "$1.merged" || - die "the fetched $2 has no deny list in its test-support rule" - mv "$1.merged" "$1" - own="$(local_only_lines "$1" "$2")" - if [ -n "$own" ]; then - die "$2 has lines the canonical file has never held; only the deny list of the test-support rule may differ from canonical: -$own" - fi -} - main() { + [ "$#" -eq 0 ] || die "usage: script/update-policy" + cd "$ROOT" - [ "$#" -le 1 ] || die "usage: script/update-policy [full commit hash]" - - project="$(script/projectname)" - if [ "$project" = "prompts" ]; then - die "this repo is the source of the policy files" - fi - - commit="${1:-}" - if [ -z "$commit" ]; then - head_line="$(git ls-remote "$PROMPTS_URL.git" refs/heads/main)" - commit="${head_line%%[[:space:]]*}" - fi - printf '%s\n' "$commit" | grep -Eqx '[0-9a-f]{40}' || - die "not a full commit hash: '$commit'" - tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT - git clone --quiet --bare "$PROMPTS_URL.git" "$tmp/prompts.git" || - die "cannot clone $PROMPTS_URL.git" + curl --fail --silent --show-error --max-time 60 \ + --output "$tmp/install" "$INSTALL_URL" || + die "cannot fetch $INSTALL_URL" - carried="" - for file in $VENDORED_FILES; do - path="$(local_path "$file")" - [ -n "$path" ] || continue - if [ -n "$(git status --porcelain -- "$path")" ]; then - die "$path has uncommitted changes" - fi - mkdir -p "$tmp/$(dirname "$path")" - curl --fail --silent --show-error --max-time 60 \ - --output "$tmp/$path" \ - "$PROMPTS_URL/raw/commit/$commit/$file" || - die "cannot fetch $file at $commit" - git -C "$tmp/prompts.git" log -p --format= "$commit" -- "$file" | - sed -n 's/^-//p' > "$tmp/$path.former" - carried="$carried $path" - done - [ -n "$carried" ] || die "this repo carries none of the vendored files" + sh "$tmp/install" "$ROOT" > "$tmp/written" + commit="$(head -n 1 "$tmp/written")" + paths="$(tail -n +2 "$tmp/written")" - for path in $carried; do - case "$path" in - .gitignore | .editorconfig | .dockerignore) - keep_lines_below_last "$tmp/$path" "$path" - ;; - .golangci.yml) - keep_deny_entries "$tmp/$path" "$path" - ;; - esac - done - - for path in $carried; do - cp "$tmp/$path" "$path" + for path in $paths; do git add -- "$path" done - if git diff --cached --quiet -- $carried; then + if git diff --cached --quiet -- $paths; then echo "update-policy: already at prompts $commit, nothing to commit" else - git commit -m "Update policy files from prompts $commit" -- $carried + git commit -m "Update policy files from prompts $commit" -- $paths fi - - # Also pushes when there was nothing to commit, so a run after a - # failed push delivers the commit that push left behind. - git push origin HEAD || die "push failed; fix that and run this again" } main "$@" diff --git a/test/update-policy.sh b/test/update-policy.sh index 08bdee5..70e1094 100755 --- a/test/update-policy.sh +++ b/test/update-policy.sh @@ -1,12 +1,14 @@ #!/bin/sh -# test/update-policy.sh: test script/update-policy on throwaway repos. -# Each scenario gets one temp directory holding a repo, its local bare -# origin, a local git repository of the canonical files in place of -# prompts, and a stub curl that serves them, so nothing touches the -# network. +# test/update-policy.sh: test script/install and script/update-policy on +# throwaway repos. Each scenario gets one temp directory holding a repo, +# its local bare origin, a local git repository of the canonical files +# in place of prompts, and a stub curl that serves them, so nothing +# touches the network. set -eu -UPDATE_POLICY="$(cd "$(dirname "$0")/.." && pwd -P)/script/update-policy" +SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd -P)/script" +INSTALL="$SCRIPT_DIR/install" +UPDATE_POLICY="$SCRIPT_DIR/update-policy" dir="" trap 'rm -rf "$dir"' EXIT @@ -17,6 +19,8 @@ fail() { exit 1 } +# The stub stops with curl's HTTP error status on any URL containing +# FAIL_URL, and answers with no commit hash when BAD_BRANCH_JSON is set. write_stub_curl() { mkdir "$dir/bin" cat > "$dir/bin/curl" << 'EOF' @@ -29,21 +33,53 @@ for argument in "$@"; do previous="$argument" done url="$previous" -base="https://git.eeqj.de/sneak/prompts/raw/commit/$HASH/" -path="${url#"$base"}" -source="$(dirname "$0")/../canonical/$path" -if [ "$path" = "$url" ] || [ ! -f "$source" ]; then +canonical="$(dirname "$0")/../canonical" +site="https://git.eeqj.de/sneak/prompts" +first="af257bef63f162a681bddc80d9b941c47a44b748" + +not_found() { [ -n "$fail_flag" ] && exit 22 echo "404 page not found" > "$output" exit 0 -fi -cp "$source" "$output" +} + +serve_file() { + [ -f "$canonical/$1" ] || not_found + cp "$canonical/$1" "$output" +} + +case "$url" in + *"${FAIL_URL:-no url contains this}"*) + not_found + ;; + "https://git.eeqj.de/api/v1/repos/sneak/prompts/branches/main") + if [ -n "${BAD_BRANCH_JSON:-}" ]; then + echo '{"message":"no such branch"}' > "$output" + else + printf '{"name":"main","commit":{"id":"%s"},"x":{"id":"%s"}}\n' \ + "$HASH" 0000000000000000000000000000000000000000 > "$output" + fi + ;; + "$site/raw/branch/main/script/install") + serve_file script/install + ;; + "$site/compare/$first...$HASH.patch") + git -C "$canonical" format-patch --stdout --root "$HASH" > "$output" + ;; + "$site/raw/commit/$HASH/"*) + serve_file "${url#"$site/raw/commit/$HASH/"}" + ;; + *) + not_found + ;; +esac EOF chmod +x "$dir/bin/curl" } write_canonical_files() { - mkdir -p "$dir/canonical/prompts" + mkdir -p "$dir/canonical/prompts" "$dir/canonical/script" + cp "$INSTALL" "$UPDATE_POLICY" "$dir/canonical/script/" cat > "$dir/canonical/.gitignore" << 'EOF' *.log *.tmp @@ -87,7 +123,7 @@ EOF # curl serves only the newest commit; every other commit gets a 404. commit_canonical() { git -C "$dir/canonical" add .gitignore .dockerignore .editorconfig \ - .golangci.yml .prettierrc prompts + .golangci.yml .prettierrc prompts script git -C "$dir/canonical" commit -q -m "Canonical" HASH="$(git -C "$dir/canonical" rev-parse HEAD)" } @@ -144,16 +180,16 @@ start_scenario() { rm -rf "$dir" dir="$(mktemp -d)" : > "$dir/output" + mkdir "$dir/tmp" write_stub_curl write_canonical_files - # git clone reads no repo's own config, so the setting that points - # git at the local canonical repository has to be in this file. export GIT_CONFIG_GLOBAL="$dir/gitconfig" git config --global user.name "Test" git config --global user.email "test@example.com" - git config --global "url.$dir/canonical.insteadOf" \ - "https://git.eeqj.de/sneak/prompts.git" + + export FAIL_URL="" + export BAD_BRANCH_JSON="" git init -q -b main "$dir/canonical" export HASH @@ -164,7 +200,7 @@ start_scenario() { git -C "$dir/repo" remote add origin "$dir/origin.git" mkdir "$dir/repo/script" - cp "$UPDATE_POLICY" "$dir/repo/script/update-policy" + cp "$INSTALL" "$UPDATE_POLICY" "$dir/repo/script/" printf '#!/bin/sh\necho example\n' > "$dir/repo/script/projectname" chmod +x "$dir/repo/script/projectname" write_outdated_files @@ -185,17 +221,16 @@ commit_and_push_fixture() { run_update_policy() { status=0 - (cd "$dir/repo" && PATH="$dir/bin:$PATH" script/update-policy "$@") \ - > "$dir/output" 2>&1 || status=$? + (cd "$dir/repo" && PATH="$dir/bin:$PATH" TMPDIR="$dir/tmp" \ + script/update-policy "$@") > "$dir/output" 2>&1 || status=$? } -reject_pushes() { - printf '#!/bin/sh\nexit 1\n' > "$dir/origin.git/hooks/pre-receive" - chmod +x "$dir/origin.git/hooks/pre-receive" -} - -accept_pushes() { - rm "$dir/origin.git/hooks/pre-receive" +# run_install : run the script in this source tree, from +# outside the repo it updates. +run_install() { + status=0 + (cd "$dir" && PATH="$dir/bin:$PATH" TMPDIR="$dir/tmp" \ + "$INSTALL" "$@") > "$dir/output" 2>&1 || status=$? } head_of_repo() { @@ -206,6 +241,10 @@ head_of_origin() { git -C "$dir/origin.git" rev-parse refs/heads/main } +files_in_commit() { + git -C "$dir/repo" show --name-only --format= HEAD | tr '\n' ' ' +} + expect_success() { [ "$status" -eq 0 ] || fail "exited $status" } @@ -230,22 +269,44 @@ expect_file() { diff "$dir/expected" "$dir/repo/$1" >&2 || fail "$1 is wrong" } +expect_same_as_canonical() { + diff "$dir/canonical/$1" "$dir/repo/$1" >&2 || fail "$1 is not canonical" +} + expect_absent() { [ ! -e "$dir/repo/$1" ] || fail "$1 exists" } +expect_executable() { + [ "$(stat -c %a "$dir/repo/$1")" = 755 ] || fail "$1 is not mode 755" +} + +expect_committed_executable() { + git -C "$dir/repo" ls-files -s -- "$1" | grep -q '^100755 ' || + fail "$1 is not committed as executable" +} + +expect_new_commit_not_pushed() { + [ "$(head_of_repo)" != "$before" ] || fail "no commit was made" + [ "$(head_of_origin)" = "$before" ] || fail "the commit was pushed" +} + +expect_no_commit() { + [ "$(head_of_repo)" = "$before" ] || fail "the repo has a new commit" + [ "$(head_of_origin)" = "$before" ] || fail "origin has a new commit" +} + expect_nothing_written() { [ -z "$(git -C "$dir/repo" status --porcelain)" ] || fail "the working tree changed" - [ "$(head_of_repo)" = "$before" ] || fail "the repo has a new commit" - [ "$(head_of_origin)" = "$before" ] || fail "origin has a new commit" + expect_no_commit } test_own_entries_kept() { start_scenario "own entries kept in all four files" commit_and_push_fixture .gitignore .dockerignore .editorconfig \ .golangci.yml .prettierrc REPO_POLICIES.md - run_update_policy "$HASH" + run_update_policy expect_success expect_file .gitignore << 'EOF' @@ -296,11 +357,12 @@ EOF new policy EOF - [ "$(head_of_origin)" = "$(head_of_repo)" ] || - fail "the commit was not pushed" - [ "$(head_of_repo)" != "$before" ] || fail "no commit was made" + expect_new_commit_not_pushed git -C "$dir/repo" log -1 --format=%s | grep -q -F -e "$HASH" || - fail "the commit message does not name the hash" + fail "the commit message does not name the head of main" + [ "$(files_in_commit)" = ".dockerignore .editorconfig .gitignore .golangci.yml .prettierrc REPO_POLICIES.md " ] || + fail "the commit holds other files: $(files_in_commit)" + [ -z "$(ls -A "$dir/tmp")" ] || fail "a temporary directory is left" } test_files_not_carried_are_skipped() { @@ -308,7 +370,7 @@ test_files_not_carried_are_skipped() { mkdir "$dir/repo/docs" mv "$dir/repo/REPO_POLICIES.md" "$dir/repo/docs/REPO_POLICIES.md" commit_and_push_fixture docs/REPO_POLICIES.md - run_update_policy "$HASH" + run_update_policy expect_success expect_file docs/REPO_POLICIES.md << 'EOF' @@ -322,40 +384,145 @@ EOF expect_absent .editorconfig expect_absent .golangci.yml expect_absent .prettierrc - [ "$(head_of_origin)" = "$(head_of_repo)" ] || - fail "the commit was not pushed" + expect_new_commit_not_pushed } test_nothing_changed() { start_scenario "nothing changed" cp "$dir/canonical/.prettierrc" "$dir/repo/.prettierrc" commit_and_push_fixture .prettierrc - run_update_policy "$HASH" + run_update_policy expect_success expect_output "nothing to commit" expect_nothing_written } -test_bad_commit_hash() { - for hash in not-a-hash 0000000000000000000000000000000000000000; do - start_scenario "bad commit hash $hash stops with nothing written" - commit_and_push_fixture .gitignore .prettierrc REPO_POLICIES.md - run_update_policy "$hash" - expect_failure - expect_nothing_written - done - # The last hash has the right form, so it is the one that reaches curl. - expect_output "cannot fetch" +test_target_lacking_install_gets_both_scripts() { + start_scenario "a repo lacking script/install gets it, and the current script/update-policy, in the commit" + rm "$dir/repo/script/install" + echo "# An older version." >> "$dir/repo/script/update-policy" + commit_and_push_fixture .prettierrc + run_update_policy + expect_success + expect_same_as_canonical script/install + expect_same_as_canonical script/update-policy + expect_committed_executable script/install + expect_committed_executable script/update-policy + expect_new_commit_not_pushed + [ "$(files_in_commit)" = ".prettierrc script/install script/update-policy " ] || + fail "the commit holds other files: $(files_in_commit)" } -test_push_failure() { - start_scenario "push failure exits non-zero" +# The shell reads a script in blocks, so the padding makes it read the +# last line only after main has returned. It finds that line only if +# script/install left the old file in place under a new name. +test_update_policy_replaces_itself() { + start_scenario "script/update-policy replaces its own file and still exits 0" + { + cat "$dir/repo/script/update-policy" + i=0 + while [ "$i" -lt 2000 ]; do + echo "# Padding." + i=$((i + 1)) + done + echo "echo 'the old file ran to its end'" + } > "$dir/merged" + mv "$dir/merged" "$dir/repo/script/update-policy" + chmod 755 "$dir/repo/script/update-policy" + commit_and_push_fixture + run_update_policy + expect_success + expect_output "the old file ran to its end" + expect_same_as_canonical script/update-policy + expect_executable script/update-policy + expect_new_commit_not_pushed +} + +test_install_with_path_commits_nothing() { + start_scenario "script/install with a path writes into that clone and commits nothing" + rm "$dir/repo/script/install" "$dir/repo/script/update-policy" commit_and_push_fixture .prettierrc - reject_pushes - run_update_policy "$HASH" + run_install "$dir/repo" + expect_success + expect_file .prettierrc << 'EOF' +{ "tabWidth": 4 } +EOF + expect_same_as_canonical script/install + expect_same_as_canonical script/update-policy + expect_executable script/install + expect_executable script/update-policy + expect_no_commit + [ "$(head -n 1 "$dir/output")" = "$HASH" ] || + fail "the first line of output is not the hash" + expect_output "script/install" + [ -z "$(ls -A "$dir/tmp")" ] || fail "a temporary directory is left" +} + +test_install_without_argument_updates_own_repo() { + start_scenario "script/install with no argument updates the repo it sits in" + commit_and_push_fixture .prettierrc + status=0 + (cd "$dir" && PATH="$dir/bin:$PATH" repo/script/install) \ + > "$dir/output" 2>&1 || status=$? + expect_success + expect_file .prettierrc << 'EOF' +{ "tabWidth": 4 } +EOF + expect_no_commit +} + +test_install_bad_target() { + start_scenario "script/install refuses a path that is not the top of a clone" + commit_and_push_fixture .prettierrc + run_install "$dir/repo/script" expect_failure - expect_output "push failed" - [ "$(head_of_origin)" = "$before" ] || fail "origin has a new commit" + expect_output "not the top of a git clone" + run_install "$dir/bin" + expect_failure + expect_output "not the top of a git clone" + run_install "$dir/no-such-directory" + expect_failure + expect_output "no such directory" + run_install "$dir/repo" extra + expect_failure + expect_output "usage:" + expect_nothing_written +} + +test_failed_fetch_stops() { + for url in \ + "/api/v1/repos/sneak/prompts/branches/main" \ + "/raw/branch/main/script/install" \ + "/compare/" \ + "/raw/commit/"; do + start_scenario "a failed fetch of $url stops with nothing written" + commit_and_push_fixture .gitignore .prettierrc REPO_POLICIES.md + FAIL_URL="$url" + run_update_policy + expect_failure + expect_output "cannot fetch" + expect_nothing_written + done +} + +test_one_missing_file_stops_all() { + start_scenario "one file missing from prompts stops before any file is written" + rm "$dir/canonical/prompts/REPO_POLICIES.md" + commit_and_push_fixture .prettierrc REPO_POLICIES.md + run_update_policy + expect_failure + expect_output "cannot fetch" + expect_nothing_written +} + +test_branch_response_without_hash_stops() { + start_scenario "a branch response with no commit hash stops" + commit_and_push_fixture .prettierrc + BAD_BRANCH_JSON=yes + run_update_policy + expect_failure + expect_output "no commit hash" + expect_nothing_written } # Each of these files, with the repo's own entries above the closing @@ -370,7 +537,7 @@ test_own_entry_above_closing_comment() { mv "$dir/merged" "$dir/repo/$file" commit_and_push_fixture .gitignore .dockerignore .editorconfig \ .prettierrc REPO_POLICIES.md - run_update_policy "$HASH" + run_update_policy expect_failure expect_output "$file has entries" expect_output "/myapp" @@ -380,44 +547,39 @@ test_own_entry_above_closing_comment() { done } -test_retry_after_failed_push() { - start_scenario "a retry after a failed push delivers the commit" - commit_and_push_fixture .prettierrc - reject_pushes - run_update_policy "$HASH" - expect_failure - committed="$(head_of_repo)" - [ "$committed" != "$before" ] || fail "no commit was made" - [ "$(head_of_origin)" = "$before" ] || fail "origin has a new commit" - - accept_pushes - run_update_policy "$HASH" - expect_success - [ "$(head_of_repo)" = "$committed" ] || fail "the retry made a new commit" - [ "$(head_of_origin)" = "$committed" ] || - fail "the retry did not deliver the commit" -} - test_uncommitted_change_stops() { start_scenario "a carried file with uncommitted changes stops" commit_and_push_fixture .prettierrc REPO_POLICIES.md echo "uncommitted" >> "$dir/repo/REPO_POLICIES.md" - run_update_policy "$HASH" + run_update_policy expect_failure expect_output "REPO_POLICIES.md has uncommitted changes" expect_file REPO_POLICIES.md << 'EOF' old policy uncommitted EOF - [ "$(head_of_repo)" = "$before" ] || fail "the repo has a new commit" - [ "$(head_of_origin)" = "$before" ] || fail "origin has a new commit" + expect_no_commit +} + +test_untracked_script_stops() { + start_scenario "an untracked script/install is not overwritten" + rm "$dir/repo/script/install" + commit_and_push_fixture .prettierrc + echo "our own script" > "$dir/repo/script/install" + run_install "$dir/repo" + expect_failure + expect_output "script/install has uncommitted changes" + expect_file script/install << 'EOF' +our own script +EOF + expect_no_commit } test_gitignore_lacks_canonical_last_line() { start_scenario ".gitignore lacking the canonical last line stops" printf '*.log\n/myapp\n' > "$dir/repo/.gitignore" commit_and_push_fixture .gitignore .prettierrc - run_update_policy "$HASH" + run_update_policy expect_failure expect_output ".gitignore lacks this line" expect_nothing_written @@ -438,7 +600,7 @@ linters: - pkg: example.com/other EOF commit_and_push_fixture .golangci.yml .prettierrc - run_update_policy "$HASH" + run_update_policy expect_failure expect_output "no deny list in its test-support rule" expect_nothing_written @@ -453,7 +615,7 @@ test_own_golangci_setting_stops() { - pattern: model.Operator EOF commit_and_push_fixture .golangci.yml .prettierrc - run_update_policy "$HASH" + run_update_policy expect_failure expect_output ".golangci.yml has lines" expect_output "forbidigo:" @@ -464,9 +626,10 @@ EOF test_source_repo_stops() { start_scenario "the repo the policy files come from stops" - printf '#!/bin/sh\necho prompts\n' > "$dir/repo/script/projectname" - commit_and_push_fixture .prettierrc REPO_POLICIES.md - run_update_policy "$HASH" + mkdir "$dir/repo/prompts" + echo "policy" > "$dir/repo/prompts/REPO_POLICIES.md" + commit_and_push_fixture .prettierrc prompts/REPO_POLICIES.md + run_update_policy expect_failure expect_output "this repo is the source of the policy files" expect_nothing_written @@ -477,42 +640,26 @@ test_staged_file_stays_out_of_commit() { commit_and_push_fixture .prettierrc echo "notes" > "$dir/repo/notes.txt" git -C "$dir/repo" add notes.txt - run_update_policy "$HASH" + run_update_policy expect_success - [ "$(git -C "$dir/repo" show --name-only --format= HEAD)" = ".prettierrc" ] || + [ "$(files_in_commit)" = ".prettierrc " ] || fail "the commit holds more than the policy file" [ "$(git -C "$dir/repo" diff --cached --name-only)" = "notes.txt" ] || fail "notes.txt is no longer staged" } -test_short_commit_hash_refused() { - start_scenario "a hexadecimal hash shorter than 40 characters is refused" +test_update_policy_takes_no_argument() { + start_scenario "script/update-policy refuses an argument" commit_and_push_fixture .prettierrc - run_update_policy "${HASH%?}" + run_update_policy "$HASH" expect_failure - expect_output "not a full commit hash" + expect_output "usage:" expect_nothing_written } -# git asks the local canonical repository for the head of main, and the -# stub curl serves only that commit. -test_no_hash_uses_head_of_main() { - start_scenario "with no hash the script uses the head of main" - commit_and_push_fixture .prettierrc - run_update_policy - expect_success - expect_file .prettierrc << 'EOF' -{ "tabWidth": 4 } -EOF - git -C "$dir/repo" log -1 --format=%s | grep -q -F -e "$HASH" || - fail "the commit message does not name the head of main" - [ "$(head_of_origin)" = "$(head_of_repo)" ] || - fail "the commit was not pushed" -} - # expect_former_canonical_line_replaced : an earlier # canonical commit held the line and the newest does not. This repo's -# file still has it, and the script replaces it and pushes. +# file still has it, and the script replaces it. expect_former_canonical_line_replaced() { cp "$dir/canonical/$1" "$dir/newest" { echo "$2"; cat "$dir/newest"; } > "$dir/canonical/$1" @@ -523,14 +670,12 @@ expect_former_canonical_line_replaced() { mv "$dir/merged" "$dir/repo/$1" commit_and_push_fixture "$1" .prettierrc - run_update_policy "$HASH" + run_update_policy expect_success if grep -q -F -x -e "$2" "$dir/repo/$1"; then fail "$1 still has: $2" fi - [ "$(head_of_repo)" != "$before" ] || fail "no commit was made" - [ "$(head_of_origin)" = "$(head_of_repo)" ] || - fail "the commit was not pushed" + expect_new_commit_not_pushed } test_former_canonical_golangci_line_replaced() { @@ -543,24 +688,49 @@ test_former_canonical_gitignore_line_replaced() { expect_former_canonical_line_replaced .gitignore "*.old" } +# A line another file's history removed does not count as held by this +# one. +test_line_removed_from_another_file_still_stops() { + start_scenario "a line only another canonical file once held stops" + { echo "*.old"; cat "$dir/canonical/.dockerignore"; } > "$dir/merged" + cp "$dir/merged" "$dir/canonical/.dockerignore" + commit_canonical + tail -n +2 "$dir/merged" > "$dir/canonical/.dockerignore" + commit_canonical + { echo "*.old"; cat "$dir/repo/.gitignore"; } > "$dir/merged" + mv "$dir/merged" "$dir/repo/.gitignore" + commit_and_push_fixture .gitignore .prettierrc + run_update_policy + expect_failure + expect_output ".gitignore has entries" + expect_output "*.old" + expect_nothing_written +} + main() { test_own_entries_kept test_files_not_carried_are_skipped test_nothing_changed - test_bad_commit_hash - test_push_failure + test_target_lacking_install_gets_both_scripts + test_update_policy_replaces_itself + test_install_with_path_commits_nothing + test_install_without_argument_updates_own_repo + test_install_bad_target + test_failed_fetch_stops + test_one_missing_file_stops_all + test_branch_response_without_hash_stops test_own_entry_above_closing_comment - test_retry_after_failed_push test_uncommitted_change_stops + test_untracked_script_stops test_gitignore_lacks_canonical_last_line test_fetched_golangci_without_deny_list test_own_golangci_setting_stops test_source_repo_stops test_staged_file_stays_out_of_commit - test_short_commit_hash_refused - test_no_hash_uses_head_of_main + test_update_policy_takes_no_argument test_former_canonical_golangci_line_replaced test_former_canonical_gitignore_line_replaced + test_line_removed_from_another_file_still_stops echo "update-policy tests passed" }