diff --git a/README.md b/README.md index 4ea8801..b80ae90 100644 --- a/README.md +++ b/README.md @@ -156,10 +156,9 @@ requests. The "Library reference" below covers both. `quak backup` never prompts, so cron can run it. `make install` builds quak as one binary and copies it to `~/bin/quak`. It uses bun, a JavaScript runtime, to bundle the built `dist/` into one executable. Besides what Getting Started -needs, `make install` needs `make`. On Linux it downloads bun from GitHub with -`curl` and unpacks it with `unzip`. It checks the download against a SHA-256 -hash in the repo. On macOS it runs bun through `nix-shell -p bun`, so it needs -[nix](https://nixos.org/download). +needs, `make install` needs `make`. It downloads bun from GitHub with `curl` and +unpacks it with `unzip`. It checks the download against a SHA-256 hash in the +repo. `make install` works on Linux and macOS, on x86-64 and arm64. On Linux systems that use the musl C library, such as Alpine, `make install` downloads bun's musl build. On Alpine, both that bun and `~/bin/quak` need the @@ -1937,11 +1936,9 @@ These apply to people and to coding agents alike. - container images by digest; - the nvm archive `script/bootstrap` downloads, and the Python packages `bin/clip-keyword-vectors` installs, by SHA-256 hash; - - bun's glibc and musl builds, which `script/build-bin` downloads on Linux, - by SHA-256 hash; + - the bun builds `script/build-bin` downloads, by SHA-256 hash; - Node and yarn, when `script/bootstrap` installs them, by version. - Not pinned: - - bun on macOS, which `script/build-bin` takes from `nix-shell -p bun`; - `make` and `git`, which `script/bootstrap` installs from the system package manager when they are missing, as every `Dockerfile` stage does; - `bash` and `curl`, which `script/bootstrap` installs the same way when it diff --git a/script/build-bin b/script/build-bin index 08f72e8..418598a 100755 --- a/script/build-bin +++ b/script/build-bin @@ -3,12 +3,13 @@ # executable, bin/quak, which reports the version script/build stamped. Our own # extension to scripts-to-rule-them-all. # -# bun compiles the binary from a copy of its own executable. On Linux, nix's -# bun loads through a C library in the nix store, and so does a binary compiled -# from it. nix-collect-garbage can delete that library, and the binary then no -# longer starts. On Linux this script therefore compiles with bun's own release -# build, which loads through the system's C library. It downloads that build -# and checks it against the hash below before running it. +# bun compiles the binary from a copy of its own executable, so the binary +# loads the same libraries as the bun that built it. nix's bun loads libraries +# from the nix store: on Linux its C library, on macOS its ICU. A binary +# compiled from it loads them too. nix-collect-garbage can delete them, and the +# binary then no longer starts. This script therefore compiles with bun's own +# release build, which loads only the system's libraries. It downloads that +# build and checks it against the hash below before running it. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -22,6 +23,8 @@ BUN_LINUX_X64_BASELINE_SHA256="9d8a24292a7068090205daac0a5a223f5f69736f5287e37bf BUN_LINUX_X64_MUSL_BASELINE_SHA256="88ca7c7ad235b498f549eea2f770f434e9f0f5e9ba95168a2d3a1f235184c394" BUN_LINUX_AARCH64_SHA256="70bae41b3908b0a120e1e58c5c8af30e74afae3b8d11b0d3fdd8e787ddfb4b22" BUN_LINUX_AARCH64_MUSL_SHA256="5385e978107ce4934298d8d6afe9bfbb898683f6cc23e6753a0da60bc60c5b81" +BUN_DARWIN_X64_BASELINE_SHA256="a98ba6a480f22fda9b343626b906a4e26aa53618bf85d2bc5928ecf2ba45f0ed" +BUN_DARWIN_AARCH64_SHA256="5467e3f65dba526b9fea98f0cce04efafc0c63e169733ec27b876a3ad32da190" # compile_with_downloaded_bun compile_with_downloaded_bun() { @@ -29,7 +32,12 @@ compile_with_downloaded_bun() { trap 'rm -rf "$tmp"' EXIT curl -fsSL -o "$tmp/$1.zip" \ "https://github.com/oven-sh/bun/releases/download/bun-v$BUN_VERSION/$1.zip" - echo "$2 $tmp/$1.zip" | sha256sum -c - + # macOS always has shasum, but not every macOS has sha256sum. + if command -v sha256sum >/dev/null 2>&1; then + echo "$2 $tmp/$1.zip" | sha256sum -c - + else + echo "$2 $tmp/$1.zip" | shasum -a 256 -c - + fi unzip -q "$tmp/$1.zip" -d "$tmp" "$tmp/$1/bun" build dist/bin/quak.js --compile --outfile bin/quak } @@ -59,13 +67,17 @@ main() { compile_with_downloaded_bun bun-linux-aarch64-musl \ "$BUN_LINUX_AARCH64_MUSL_SHA256" ;; - Linux-*) - echo "build-bin: bun has no release build for Linux on $(uname -m)" >&2 - exit 1 + Darwin-x86_64) + compile_with_downloaded_bun bun-darwin-x64-baseline \ + "$BUN_DARWIN_X64_BASELINE_SHA256" + ;; + Darwin-arm64) + compile_with_downloaded_bun bun-darwin-aarch64 \ + "$BUN_DARWIN_AARCH64_SHA256" ;; *) - nix-shell -p bun --run \ - "bun build dist/bin/quak.js --compile --outfile bin/quak" + echo "build-bin: bun has no release build for $(uname -s) on $(uname -m)" >&2 + exit 1 ;; esac } diff --git a/test/packaging/build-bin.test.ts b/test/packaging/build-bin.test.ts index d1f96ad..6f7a73f 100644 --- a/test/packaging/build-bin.test.ts +++ b/test/packaging/build-bin.test.ts @@ -1,30 +1,29 @@ // `script/build-bin` compiles `bin/quak`, the single binary that the README's // cron setup runs. bun compiles it from a copy of its own executable, so the -// binary loads through the same C library as the bun that built it. nix's bun -// loads through a C library in the nix store, which garbage collection can -// delete. On Linux the script therefore downloads bun's own release build, -// which loads through the system's C library. +// binary loads the same libraries as the bun that built it. nix's bun loads +// libraries from the nix store, which garbage collection can delete: on Linux +// its C library, on macOS its ICU. The script therefore downloads bun's own +// release build, which loads only the system's libraries. // // What the script promises, and why each test matters: // -// - On Linux, each machine and C library that bun publishes a build for gets -// its own archive: x86-64 or arm64, with glibc or with musl, as on Alpine. A -// bun built for another machine or C library does not start, and -// `make install` fails. +// - Each system, machine and C library that bun publishes a build for gets its +// own archive: Linux on x86-64 or arm64, with glibc or with musl, as on +// Alpine, and macOS on Intel or Apple silicon. A bun built for another +// system, machine or C library does not start, and `make install` fails. // - The archive's SHA-256 must match the one in the script before anything in // the archive runs. Without that check, a tampered download would build the // binary that cron runs. -// - Linux on any other machine stops with an error. Building there with nix's -// bun would bring back the dependency on the nix store. -// - Other systems, such as macOS, compile with nix's bun as before, and -// download nothing. +// - Any other system or machine stops with an error, and never falls back to +// nix's bun. Building with nix's bun would bring back the dependency on the +// nix store. // // Each test copies the script into a fresh directory, which the script then // treats as the checkout, and runs it there. Stubs first on PATH stand in for -// `uname`, `ldd`, `curl`, `unzip` and `nix-shell`, so no test downloads -// anything or runs a real bun. +// `uname`, `ldd`, `curl`, `unzip`, `nix-shell` and `shasum`, so no test +// downloads anything or runs a real bun. import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { spawnSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import { chmodSync, copyFileSync, @@ -33,6 +32,7 @@ import { mkdtempSync, readFileSync, rmSync, + symlinkSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -42,9 +42,10 @@ import { fileURLToPath } from "node:url"; const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); // The stubs, keyed by the command each replaces. `uname` and `ldd` describe the -// system that runBuildBin passes in the environment. `curl`, `unzip` and -// `nix-shell` write the arguments they were given, one per line, to a file -// named after the command in $STUB_LOG_DIR, which also shows that they ran. +// system that runBuildBin passes in the environment. `curl`, `unzip`, +// `nix-shell` and `shasum` write the arguments they were given, one per line, +// to a file named after the command in $STUB_LOG_DIR, which also shows that +// they ran. const stubs: Record = { uname: ` case "$1" in @@ -82,6 +83,11 @@ const stubs: Record = { chmod +x "$bun"`, "nix-shell": ` printf '%s\\n' "$@" > "$STUB_LOG_DIR/nix-shell"`, + // The script checks the hash with `shasum` where there is no `sha256sum`. + // The stub fails, as `shasum` does when the hash does not match. + shasum: ` + printf '%s\\n' "$@" > "$STUB_LOG_DIR/shasum" + exit 1`, }; let checkout = ""; @@ -114,13 +120,21 @@ afterEach(() => { }); // Runs the script with nothing in its environment but the stubs' settings and a -// PATH that finds the stubs first. -const runBuildBin = (system: string, machine: string, libc = "glibc") => +// PATH. Unless the caller passes one, the PATH finds the stubs first and then +// the host's commands. +const runBuildBin = ( + system: string, + machine: string, + { + libc = "glibc", + path = `${join(checkout, "stubs")}:${process.env.PATH}`, + } = {}, +) => spawnSync(join(checkout, "script/build-bin"), { cwd: checkout, encoding: "utf-8", env: { - PATH: `${join(checkout, "stubs")}:${process.env.PATH}`, + PATH: path, STUB_UNAME_S: system, STUB_UNAME_M: machine, STUB_LIBC: libc, @@ -128,6 +142,31 @@ const runBuildBin = (system: string, machine: string, libc = "glibc") => }, }); +// The real commands that the script and the stubs run. +const hostCommands = [ + "basename", + "chmod", + "dirname", + "grep", + "mkdir", + "mktemp", + "rm", +]; + +// A Mac has no `ldd`, and not every Mac has `sha256sum`. This makes a PATH that +// finds neither: the stubs other than `ldd`, then links to hostCommands. +const makeMacPath = (): string => { + rmSync(join(checkout, "stubs", "ldd")); + mkdirSync(join(checkout, "host-commands")); + for (const command of hostCommands) { + const target = execFileSync("sh", ["-c", `command -v ${command}`], { + encoding: "utf-8", + }).trim(); + symlinkSync(target, join(checkout, "host-commands", command)); + } + return `${join(checkout, "stubs")}:${join(checkout, "host-commands")}`; +}; + const ran = (command: string): boolean => existsSync(join(checkout, "log", command)); @@ -139,42 +178,64 @@ const argumentsOf = (command: string): string[] => describe("script/build-bin", () => { // The stub curl writes a file that is not the archive, so the hash check // must stop the script before unzip runs. A script without the check would - // run unzip, and the stub bun it unpacks would write bin/quak. + // run unzip, and the stub bun it unpacks would write bin/quak. On Linux the + // check must use `sha256sum`, since a stock Alpine has no `shasum`. it.each([ - ["x86_64", "glibc", "bun-linux-x64-baseline.zip"], - ["x86_64", "musl", "bun-linux-x64-musl-baseline.zip"], - ["aarch64", "glibc", "bun-linux-aarch64.zip"], - ["aarch64", "musl", "bun-linux-aarch64-musl.zip"], + ["Linux", "x86_64", "glibc", "bun-linux-x64-baseline.zip"], + ["Linux", "x86_64", "musl", "bun-linux-x64-musl-baseline.zip"], + ["Linux", "aarch64", "glibc", "bun-linux-aarch64.zip"], + ["Linux", "aarch64", "musl", "bun-linux-aarch64-musl.zip"], ])( - "on Linux %s with %s, downloads %s and stops when its hash does not match", - (machine, libc, archive) => { - const result = runBuildBin("Linux", machine, libc); + "on %s %s with %s, downloads %s and stops when its hash does not match", + (system, machine, libc, archive) => { + const result = runBuildBin(system, machine, { libc }); expect(argumentsOf("curl")).toContain( "https://github.com/oven-sh/bun/releases/download/bun-v1.3.13/" + archive, ); + expect(ran("shasum")).toBe(false); expect(result.status).not.toBe(0); expect(ran("unzip")).toBe(false); + expect(ran("nix-shell")).toBe(false); expect(existsSync(join(checkout, "bin/quak"))).toBe(false); }, ); - it("stops on a Linux machine that bun publishes no build for", () => { - const result = runBuildBin("Linux", "riscv64"); - expect(result.status).not.toBe(0); - expect(ran("curl")).toBe(false); - expect(ran("nix-shell")).toBe(false); - }); + // On the PATH from makeMacPath the script finds no `sha256sum` and checks + // the hash with the stub `shasum`. That stub fails, so the script must stop + // before unzip runs. + // On Apple silicon `uname -m` says arm64, where bun's archive says aarch64. + it.each([ + ["Darwin", "x86_64", "bun-darwin-x64-baseline.zip"], + ["Darwin", "arm64", "bun-darwin-aarch64.zip"], + ])( + "on %s %s, downloads %s and stops when shasum rejects its hash", + (system, machine, archive) => { + const result = runBuildBin(system, machine, { + path: makeMacPath(), + }); + expect(argumentsOf("curl")).toContain( + "https://github.com/oven-sh/bun/releases/download/bun-v1.3.13/" + + archive, + ); + expect(argumentsOf("shasum")).toEqual(["-a", "256", "-c", "-"]); + expect(result.status).not.toBe(0); + expect(ran("unzip")).toBe(false); + expect(ran("nix-shell")).toBe(false); + expect(existsSync(join(checkout, "bin/quak"))).toBe(false); + }, + ); - it("compiles with nix's bun on macOS", () => { - const result = runBuildBin("Darwin", "arm64"); - expect(result.status).toBe(0); - expect(ran("curl")).toBe(false); - expect(argumentsOf("nix-shell")).toEqual([ - "-p", - "bun", - "--run", - "bun build dist/bin/quak.js --compile --outfile bin/quak", - ]); - }); + it.each([ + ["Linux", "riscv64"], + ["FreeBSD", "amd64"], + ])( + "stops on %s %s, which bun publishes no build for", + (system, machine) => { + const result = runBuildBin(system, machine); + expect(result.status).not.toBe(0); + expect(ran("curl")).toBe(false); + expect(ran("nix-shell")).toBe(false); + }, + ); });