From 674fdeb482e4a0692b8052884d51a5fba4181d20 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 11 Nov 2026 20:33:28 +0000 Subject: [PATCH 1/2] make build-bin: on Linux, build with bun's pinned release build (closes #240) nix's bun loads through a glibc in the nix store, and bun build --compile copied that loader into bin/quak, so nix-collect-garbage could break the cron backup. The build moves into script/build-bin. On Linux it downloads bun 1.3.13's release archive for the machine from GitHub: x86-64 (baseline) or arm64, glibc or musl. It checks the archive's SHA-256 before running it, and compiles with that bun, so the binary loads through the system loader. macOS still runs bun through nix-shell. Model: opus-5-5 --- Makefile | 6 ++-- README.md | 23 ++++++++++----- script/build-bin | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 11 deletions(-) create mode 100755 script/build-bin diff --git a/Makefile b/Makefile index a71e286..5b04bad 100644 --- a/Makefile +++ b/Makefile @@ -26,10 +26,8 @@ check: build: @script/build -# Bundles the built dist/, so the binary reports the version script/build -# stamped. -build-bin: build - nix-shell -p bun --run "bun build dist/bin/quak.js --compile --outfile bin/quak" +build-bin: + @script/build-bin install: build-bin mkdir -p ~/bin diff --git a/README.md b/README.md index af2ef02..4ea8801 100644 --- a/README.md +++ b/README.md @@ -154,10 +154,16 @@ requests. The "Library reference" below covers both. ## Running the backup from cron `quak backup` never prompts, so cron can run it. `make install` builds quak as -one binary and copies it to `~/bin/quak`. Besides what Getting Started needs, -`make install` needs `make` and [nix](https://nixos.org/download). It uses bun, -a JavaScript runtime, to bundle the built `dist/` into one executable. It runs -bun through `nix-shell -p bun`. +one binary and copies it to `~/bin/quak`. It uses bun, a JavaScript runtime, to +bundle the built `dist/` into one executable. Besides what Getting Started +needs, `make install` needs `make`. On Linux it downloads bun from GitHub with +`curl` and unpacks it with `unzip`. It checks the download against a SHA-256 +hash in the repo. On macOS it runs bun through `nix-shell -p bun`, so it needs +[nix](https://nixos.org/download). + +On Linux systems that use the musl C library, such as Alpine, `make install` +downloads bun's musl build. On Alpine, both that bun and `~/bin/quak` need the +`libstdc++` and `libgcc` packages. Log in once with `~/bin/quak`, as the user the cron job will run as: @@ -1793,6 +1799,8 @@ standard. The others are additions to it. `package.json` declares (`main`, `types`, `bin`) were written, and make the command line executable. - `script/version`: print the version `script/build` stamps (see "Version"). +- `script/build-bin`: run `script/build`, then bundle the built `dist/` into one + binary, `bin/quak`, with bun (see "Running the backup from cron"). - `script/test`: run the test suite, by building the `test` phase of the `Dockerfile`. Requires Docker. - `script/lint`: run eslint and a prettier check, by building the `lint` phase @@ -1821,8 +1829,7 @@ The Makefile targets are: - `make build`: runs `script/build`. - `make docker`: runs `script/docker`. - `make hooks`: runs `script/install-precommit`. -- `make build-bin`: bundles the built `dist/` into one binary, `bin/quak`. It - runs bun through `nix-shell -p bun`. +- `make build-bin`: runs `script/build-bin`. - `make install`: runs `make build-bin` and copies the binary to `~/bin/quak`. - `make dev`: runs the TypeScript compiler in watch mode. - `make clean`: removes `dist/`, `coverage/`, `.vitest-cache/` and the @@ -1930,9 +1937,11 @@ These apply to people and to coding agents alike. - container images by digest; - the nvm archive `script/bootstrap` downloads, and the Python packages `bin/clip-keyword-vectors` installs, by SHA-256 hash; + - bun's glibc and musl builds, which `script/build-bin` downloads on Linux, + by SHA-256 hash; - Node and yarn, when `script/bootstrap` installs them, by version. - Not pinned: - - bun, which `make build-bin` takes from `nix-shell -p bun`; + - bun on macOS, which `script/build-bin` takes from `nix-shell -p bun`; - `make` and `git`, which `script/bootstrap` installs from the system package manager when they are missing, as every `Dockerfile` stage does; - `bash` and `curl`, which `script/bootstrap` installs the same way when it diff --git a/script/build-bin b/script/build-bin new file mode 100755 index 0000000..08f72e8 --- /dev/null +++ b/script/build-bin @@ -0,0 +1,73 @@ +#!/bin/sh +# script/build-bin: run script/build, then bundle the built dist/ into one +# executable, bin/quak, which reports the version script/build stamped. Our own +# extension to scripts-to-rule-them-all. +# +# bun compiles the binary from a copy of its own executable. On Linux, nix's +# bun loads through a C library in the nix store, and so does a binary compiled +# from it. nix-collect-garbage can delete that library, and the binary then no +# longer starts. On Linux this script therefore compiles with bun's own release +# build, which loads through the system's C library. It downloads that build +# and checks it against the hash below before running it. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# bun 1.3.13, 2026-11-11, and the SHA-256 of each release archive below. The +# x86-64 archives are the baseline builds, which also run on CPUs without AVX2. +# The musl archives are for Linux systems that use the musl C library, such as +# Alpine. +BUN_VERSION="1.3.13" +BUN_LINUX_X64_BASELINE_SHA256="9d8a24292a7068090205daac0a5a223f5f69736f5287e37bf88d3b4031edc750" +BUN_LINUX_X64_MUSL_BASELINE_SHA256="88ca7c7ad235b498f549eea2f770f434e9f0f5e9ba95168a2d3a1f235184c394" +BUN_LINUX_AARCH64_SHA256="70bae41b3908b0a120e1e58c5c8af30e74afae3b8d11b0d3fdd8e787ddfb4b22" +BUN_LINUX_AARCH64_MUSL_SHA256="5385e978107ce4934298d8d6afe9bfbb898683f6cc23e6753a0da60bc60c5b81" + +# compile_with_downloaded_bun +compile_with_downloaded_bun() { + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' EXIT + curl -fsSL -o "$tmp/$1.zip" \ + "https://github.com/oven-sh/bun/releases/download/bun-v$BUN_VERSION/$1.zip" + echo "$2 $tmp/$1.zip" | sha256sum -c - + unzip -q "$tmp/$1.zip" -d "$tmp" + "$tmp/$1/bun" build dist/bin/quak.js --compile --outfile bin/quak +} + +main() { + cd "$ROOT" + "$ROOT/script/build" + platform="$(uname -s)-$(uname -m)" + # musl's ldd begins its version output with "musl libc"; glibc's does not. + if ldd --version 2>&1 | grep -q '^musl libc'; then + platform="$platform-musl" + fi + case "$platform" in + Linux-x86_64) + compile_with_downloaded_bun bun-linux-x64-baseline \ + "$BUN_LINUX_X64_BASELINE_SHA256" + ;; + Linux-x86_64-musl) + compile_with_downloaded_bun bun-linux-x64-musl-baseline \ + "$BUN_LINUX_X64_MUSL_BASELINE_SHA256" + ;; + Linux-aarch64) + compile_with_downloaded_bun bun-linux-aarch64 \ + "$BUN_LINUX_AARCH64_SHA256" + ;; + Linux-aarch64-musl) + compile_with_downloaded_bun bun-linux-aarch64-musl \ + "$BUN_LINUX_AARCH64_MUSL_SHA256" + ;; + Linux-*) + echo "build-bin: bun has no release build for Linux on $(uname -m)" >&2 + exit 1 + ;; + *) + nix-shell -p bun --run \ + "bun build dist/bin/quak.js --compile --outfile bin/quak" + ;; + esac +} + +main "$@" -- 2.54.0 From 685f1d584c8ecff0569b4ea50b8fee959cac2dee Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 11 Nov 2026 22:52:58 +0000 Subject: [PATCH 2/2] Test script/build-bin's archive choice, hash check and refusal The test runs the script with stub uname, ldd, curl, unzip and nix-shell first on PATH, so it needs no network and no bun. The download and the compile stay untested. Model: opus-5-5 --- test/packaging/build-bin.test.ts | 180 +++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 test/packaging/build-bin.test.ts diff --git a/test/packaging/build-bin.test.ts b/test/packaging/build-bin.test.ts new file mode 100644 index 0000000..d1f96ad --- /dev/null +++ b/test/packaging/build-bin.test.ts @@ -0,0 +1,180 @@ +// `script/build-bin` compiles `bin/quak`, the single binary that the README's +// cron setup runs. bun compiles it from a copy of its own executable, so the +// binary loads through the same C library as the bun that built it. nix's bun +// loads through a C library in the nix store, which garbage collection can +// delete. On Linux the script therefore downloads bun's own release build, +// which loads through the system's C library. +// +// What the script promises, and why each test matters: +// +// - On Linux, each machine and C library that bun publishes a build for gets +// its own archive: x86-64 or arm64, with glibc or with musl, as on Alpine. A +// bun built for another machine or C library does not start, and +// `make install` fails. +// - The archive's SHA-256 must match the one in the script before anything in +// the archive runs. Without that check, a tampered download would build the +// binary that cron runs. +// - Linux on any other machine stops with an error. Building there with nix's +// bun would bring back the dependency on the nix store. +// - Other systems, such as macOS, compile with nix's bun as before, and +// download nothing. +// +// Each test copies the script into a fresh directory, which the script then +// treats as the checkout, and runs it there. Stubs first on PATH stand in for +// `uname`, `ldd`, `curl`, `unzip` and `nix-shell`, so no test downloads +// anything or runs a real bun. +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { spawnSync } from "node:child_process"; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); + +// The stubs, keyed by the command each replaces. `uname` and `ldd` describe the +// system that runBuildBin passes in the environment. `curl`, `unzip` and +// `nix-shell` write the arguments they were given, one per line, to a file +// named after the command in $STUB_LOG_DIR, which also shows that they ran. +const stubs: Record = { + uname: ` + case "$1" in + -s) echo "$STUB_UNAME_S" ;; + -m) echo "$STUB_UNAME_M" ;; + esac`, + // The script looks only at the start of the line: "musl libc" or not. + ldd: ` + if [ "$STUB_LIBC" = musl ]; then + echo "musl libc (x86_64)" + else + echo "ldd (GNU libc) 2.36" + fi`, + // Writes a file that is not the archive to the path after -o. + curl: ` + printf '%s\\n' "$@" > "$STUB_LOG_DIR/curl" + while [ $# -gt 1 ]; do + if [ "$1" = -o ]; then echo "not the archive" > "$2"; fi + shift + done`, + // Unpacks a stub bun where the real archive puts bun: in a directory named + // after the archive, inside the directory after -d. That bun writes + // bin/quak, as a real compile would. + unzip: ` + printf '%s\\n' "$@" > "$STUB_LOG_DIR/unzip" + for arg in "$@"; do + case "$arg" in *.zip) archive="$arg" ;; esac + if [ "$previous" = -d ]; then directory="$arg"; fi + previous="$arg" + done + bun="$directory/$(basename "$archive" .zip)/bun" + mkdir -p "$(dirname "$bun")" + echo '#!/bin/sh' > "$bun" + echo 'mkdir -p bin && echo binary > bin/quak' >> "$bun" + chmod +x "$bun"`, + "nix-shell": ` + printf '%s\\n' "$@" > "$STUB_LOG_DIR/nix-shell"`, +}; + +let checkout = ""; + +const writeScript = (path: string, body: string): void => { + writeFileSync(path, `#!/bin/sh\n${body}\n`); + chmodSync(path, 0o755); +}; + +// A checkout holding the script, a stub script/build that does nothing, the +// stubs, and the directory they log to. +beforeEach(() => { + checkout = mkdtempSync(join(tmpdir(), "quak-build-bin-")); + mkdirSync(join(checkout, "script")); + copyFileSync( + join(repoRoot, "script/build-bin"), + join(checkout, "script/build-bin"), + ); + chmodSync(join(checkout, "script/build-bin"), 0o755); + writeScript(join(checkout, "script/build"), "exit 0"); + mkdirSync(join(checkout, "stubs")); + for (const [command, body] of Object.entries(stubs)) { + writeScript(join(checkout, "stubs", command), body); + } + mkdirSync(join(checkout, "log")); +}); + +afterEach(() => { + rmSync(checkout, { recursive: true, force: true }); +}); + +// Runs the script with nothing in its environment but the stubs' settings and a +// PATH that finds the stubs first. +const runBuildBin = (system: string, machine: string, libc = "glibc") => + spawnSync(join(checkout, "script/build-bin"), { + cwd: checkout, + encoding: "utf-8", + env: { + PATH: `${join(checkout, "stubs")}:${process.env.PATH}`, + STUB_UNAME_S: system, + STUB_UNAME_M: machine, + STUB_LIBC: libc, + STUB_LOG_DIR: join(checkout, "log"), + }, + }); + +const ran = (command: string): boolean => + existsSync(join(checkout, "log", command)); + +const argumentsOf = (command: string): string[] => + readFileSync(join(checkout, "log", command), "utf-8") + .trimEnd() + .split("\n"); + +describe("script/build-bin", () => { + // The stub curl writes a file that is not the archive, so the hash check + // must stop the script before unzip runs. A script without the check would + // run unzip, and the stub bun it unpacks would write bin/quak. + it.each([ + ["x86_64", "glibc", "bun-linux-x64-baseline.zip"], + ["x86_64", "musl", "bun-linux-x64-musl-baseline.zip"], + ["aarch64", "glibc", "bun-linux-aarch64.zip"], + ["aarch64", "musl", "bun-linux-aarch64-musl.zip"], + ])( + "on Linux %s with %s, downloads %s and stops when its hash does not match", + (machine, libc, archive) => { + const result = runBuildBin("Linux", machine, libc); + expect(argumentsOf("curl")).toContain( + "https://github.com/oven-sh/bun/releases/download/bun-v1.3.13/" + + archive, + ); + expect(result.status).not.toBe(0); + expect(ran("unzip")).toBe(false); + expect(existsSync(join(checkout, "bin/quak"))).toBe(false); + }, + ); + + it("stops on a Linux machine that bun publishes no build for", () => { + const result = runBuildBin("Linux", "riscv64"); + expect(result.status).not.toBe(0); + expect(ran("curl")).toBe(false); + expect(ran("nix-shell")).toBe(false); + }); + + it("compiles with nix's bun on macOS", () => { + const result = runBuildBin("Darwin", "arm64"); + expect(result.status).toBe(0); + expect(ran("curl")).toBe(false); + expect(argumentsOf("nix-shell")).toEqual([ + "-p", + "bun", + "--run", + "bun build dist/bin/quak.js --compile --outfile bin/quak", + ]); + }); +}); -- 2.54.0