Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
16a82ff176 |
+2
-17
@@ -17,17 +17,7 @@
|
||||
# stage that compiles runs `git describe --tags --always` on .git, which
|
||||
# does not need .git/config; that file can hold a credential, such as a
|
||||
# password in a remote URL or the token the CI checkout step stores there.
|
||||
# Each submodule keeps a config with the same exposure in its git directory
|
||||
# under .git/modules/, nested again for a submodule's own submodules, or in
|
||||
# its own .git directory when it keeps one.
|
||||
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
|
||||
# `deploy/config`, `config/lib`) loses its whole git directory, because
|
||||
# `**/.git/modules/**/config` also matches that segment's directory
|
||||
# under .git/modules/. Go's version stamping then fails the build;
|
||||
# nothing leaks. Name such a submodule without that segment:
|
||||
# `git submodule add --name`.
|
||||
**/.git/config
|
||||
**/.git/modules/**/config
|
||||
.git/config
|
||||
|
||||
# Agent scratch: one full checkout of the repo per in-flight agent.
|
||||
# Anchored because it occurs once where agents run at the repo root.
|
||||
@@ -51,9 +41,7 @@
|
||||
**/[iI][dD]_[rR][sS][aA]
|
||||
**/[iI][dD]_[dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
||||
**/[iI][dD]_[eE][dD]25519
|
||||
**/[iI][dD]_[eE][dD]25519_[sS][kK]
|
||||
|
||||
# Dependencies: restored inside the image, never copied in.
|
||||
**/node_modules
|
||||
@@ -74,8 +62,5 @@
|
||||
# This repo's own host-built binary (make build).
|
||||
/bin/mfer
|
||||
|
||||
# The tools script/bootstrap installs for script/gofumpt and
|
||||
# script/generate.
|
||||
/bin/gofumpt
|
||||
# The protoc script/bootstrap unpacks for script/generate.
|
||||
/bin/protoc
|
||||
/bin/protoc-gen-go
|
||||
|
||||
@@ -10,6 +10,3 @@ insert_final_newline = true
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[*.go]
|
||||
indent_style = tab
|
||||
|
||||
@@ -4,6 +4,6 @@ jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# actions/checkout v4.2.2, 2026-02-22
|
||||
# actions/checkout v4.2.2, 2026-03-16
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
- run: script/cibuild
|
||||
|
||||
+17
-51
@@ -1,63 +1,29 @@
|
||||
# OS
|
||||
/bin/mfer
|
||||
/bin/protoc/
|
||||
/tmp
|
||||
/node_modules/
|
||||
|
||||
# Generated manifest files
|
||||
/index.mf
|
||||
|
||||
# Secrets
|
||||
.env
|
||||
.env.*
|
||||
*.key
|
||||
*.pem
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Editors
|
||||
# Editor files
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
*.bak
|
||||
.idea/
|
||||
.vscode/
|
||||
*.sublime-*
|
||||
|
||||
# Agent scratch (worktrees of this repo, created and destroyed by
|
||||
# in-flight tooling). Unanchored: .gitignore patterns already match at
|
||||
# every depth, so no prefix is wanted here. This is not a .dockerignore
|
||||
# entry and must not be given a `**/` prefix on the way into one.
|
||||
.claude/
|
||||
|
||||
# Node
|
||||
node_modules/
|
||||
|
||||
# Secrets. Unanchored like every entry above, so each matches at every
|
||||
# depth. Matching is case-sensitive on Linux, so names use character
|
||||
# ranges rather than a lowercase form that misses `Server.Key`.
|
||||
|
||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
||||
# convention. Only the templates `example.env` and `sample.env` are
|
||||
# re-included below. A repository that commits any other template adds
|
||||
# its own negation after these lines, for example `!.env.example`.
|
||||
*.[eE][nN][vV]
|
||||
.[eE][nN][vV].*
|
||||
.[eE][nN][vV][rR][cC]
|
||||
!example.env
|
||||
!sample.env
|
||||
|
||||
# Private keys and the bundles carrying them.
|
||||
*.[pP][eE][mM]
|
||||
*.[kK][eE][yY]
|
||||
*.[pP]12
|
||||
*.[pP][fF][xX]
|
||||
[iI][dD]_[rR][sS][aA]
|
||||
[iI][dD]_[dD][sS][aA]
|
||||
[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
||||
[iI][dD]_[eE][dD]25519
|
||||
[iI][dD]_[eE][dD]25519_[sS][kK]
|
||||
|
||||
# Go build and test artifacts.
|
||||
# Go build artifacts
|
||||
*.log
|
||||
*.out
|
||||
*.test
|
||||
|
||||
# This repo's own binary (make build), and the tools script/bootstrap
|
||||
# installs into bin/.
|
||||
/bin/mfer
|
||||
/bin/gofumpt
|
||||
/bin/protoc/
|
||||
/bin/protoc-gen-go
|
||||
|
||||
# A manifest generated at the repo root, and local scratch.
|
||||
/index.mf
|
||||
/tmp
|
||||
|
||||
+2
-67
@@ -10,21 +10,14 @@ run:
|
||||
|
||||
linters:
|
||||
default: all
|
||||
enable:
|
||||
# Successor to the deprecated gomodguard. Named explicitly, rather than
|
||||
# left to `default: all`, because it carries the module policy below.
|
||||
- gomodguard_v2
|
||||
disable:
|
||||
# Genuinely incompatible with project patterns
|
||||
- exhaustruct # Requires all struct fields
|
||||
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
|
||||
- depguard # Dependency allow/block lists
|
||||
- godot # Requires comments to end with periods
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
# Deprecated: the warning is attached to the old name, so it is
|
||||
# silenced by disabling that name, not by enabling the successor.
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- gomodguard # Deprecated, replaced by gomodguard_v2
|
||||
settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
@@ -35,64 +28,6 @@ linters:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
depguard:
|
||||
# Test-support code must not be compiled into the shipped binary. A
|
||||
# test-support package exists to hand a test privileges the program
|
||||
# itself must never have, so a file that is not a test must not import
|
||||
# one. Test files, and the files inside a package whose directory name
|
||||
# ends in `test`, are where that code belongs, and are exempt.
|
||||
#
|
||||
# The deny list below is the one part of this file a repository is
|
||||
# expected to extend, and the only part it may. depguard matches an
|
||||
# import path against a list of prefixes, so it cannot be told "any path
|
||||
# whose last segment ends in test"; a repository's own test-support
|
||||
# packages have to be named here one at a time, by full import path,
|
||||
# under a module path that differs from repository to repository. Add
|
||||
# them; change nothing else.
|
||||
rules:
|
||||
test-support:
|
||||
list-mode: lax
|
||||
files:
|
||||
- "$all"
|
||||
- "!$test"
|
||||
- "!**/*test/**"
|
||||
deny:
|
||||
- pkg: net/http/httptest
|
||||
desc: >-
|
||||
Test-support code belongs in test files and in packages whose
|
||||
directory name ends in test, not in the shipped binary.
|
||||
# Only decisions already recorded in the Go package defaults are
|
||||
# listed here. Every entry matches the module path exactly.
|
||||
gomodguard_v2:
|
||||
blocked:
|
||||
- module: github.com/rs/zerolog
|
||||
recommendations:
|
||||
- log/slog
|
||||
reason: "Structured logging is stdlib log/slog."
|
||||
# One entry per pre-fork module path, because the later releases
|
||||
# are separate paths. A prefix match would be shorter but would
|
||||
# also reach github.com/go-redis/redismock, the test double for
|
||||
# the successor these entries recommend.
|
||||
- module: github.com/go-redis/redis
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v7
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v8
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/sergi/go-diff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "No unified diff output; use go-udiff."
|
||||
- module: github.com/hexops/gotextdiff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "Unmaintained fork; use go-udiff."
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
|
||||
+13
-1
@@ -1,2 +1,14 @@
|
||||
# REPO_POLICIES.md is a verbatim copy of an authoritative upstream
|
||||
# document (sneak/prompts). Local tooling must never rewrite it: any
|
||||
# reformatting is silent drift from the source of truth.
|
||||
REPO_POLICIES.md
|
||||
|
||||
# User-owned configuration, copied verbatim from upstream. Not matched by
|
||||
# the current prettier file set, but listed so widening that set can never
|
||||
# start rewriting it.
|
||||
.golangci.yml
|
||||
|
||||
# Dependencies and build output
|
||||
node_modules/
|
||||
yarn.lock
|
||||
vendor/
|
||||
bin/
|
||||
|
||||
+50
-38
@@ -1,62 +1,74 @@
|
||||
# Lint phase. The linter is invoked directly rather than through `make
|
||||
# lint` or `script/lint`, which are themselves a docker build and would
|
||||
# recurse into a daemon that does not exist in a build step.
|
||||
# golangci/golangci-lint:v2.14.0, 2026-09-24
|
||||
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||
# Lint stage — fast feedback on formatting and lint issues
|
||||
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
||||
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
# Go half of fmt-check only: this image has no node, so no prettier. The
|
||||
# markdown half runs in the mdfmt stage below. The image has no gofumpt
|
||||
# either; script/gofumpt builds the version bin/tools/go.mod pins.
|
||||
RUN script/gofumpt --check
|
||||
# The linter directly, not `make lint`: script/lint builds this stage, and
|
||||
# there is no docker inside this build.
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||
# image ships and the alpine one does not.
|
||||
# golang:1.23.12, 2026-03-14
|
||||
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS test
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go test -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 90s -race -v ./...; exit 1; }
|
||||
# Markdown/JSON format stage — prettier needs node, which the Go images
|
||||
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
||||
# 22.17.0 and yarn 1.22.22, the versions .nvmrc and script/bootstrap pin.
|
||||
FROM node@sha256:b04ce4ae4e95b522112c2e5c52f781471a5cbc3b594527bcddedee9bc48c03a0 AS mdfmt
|
||||
|
||||
# Build stage. Nothing is wanted from either phase above; the copies
|
||||
# are what make BuildKit build them first, so this stage cannot run
|
||||
# unless lint and test passed. The Debian Go image ships git, which the
|
||||
# version step below needs.
|
||||
WORKDIR /src
|
||||
COPY package.json yarn.lock ./
|
||||
RUN yarn install --frozen-lockfile
|
||||
|
||||
COPY . .
|
||||
|
||||
# No make in this image; call the script entrypoint directly.
|
||||
RUN script/prettier --check
|
||||
|
||||
# Build stage — tests and compilation
|
||||
# golang:1.23.12, 2026-03-14
|
||||
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
|
||||
|
||||
# Force BuildKit to run the lint and mdfmt stages by creating stage dependencies
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /src
|
||||
COPY --from=mdfmt /src/go.sum /dev/null
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
# The revision `mfer version` prints, stamped into main.Gitrev: the
|
||||
# VERSION build arg when one is given, otherwise `git describe --tags
|
||||
# --always` on the .git in the build context. With .git present, a
|
||||
# version that is still empty, dev or unknown fails the build: git is
|
||||
# missing or could not read the checkout.
|
||||
RUN make test
|
||||
|
||||
# A build context sent as a tar archive, as upaas sends it, keeps its files'
|
||||
# owners, and git refuses to read a checkout owned by another user.
|
||||
RUN git config --system --add safe.directory /src
|
||||
|
||||
# The revision `mfer version` prints, stamped into main.Gitrev: the VERSION
|
||||
# build argument when one is given (script/docker passes one), otherwise
|
||||
# `git describe --tags --always` of the .git the build context carries: the
|
||||
# tag on a tagged commit, tag-N-gHASH on a commit after one, the short commit
|
||||
# when no tag is reachable. git ships in this base image. A context that
|
||||
# carries .git and still yields no version fails the build.
|
||||
ARG VERSION
|
||||
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||
if [ -e .git ]; then \
|
||||
case "$VERSION" in ""|dev|unknown) \
|
||||
echo "version is '$VERSION' although .git is present" >&2; \
|
||||
exit 1 ;; \
|
||||
esac; \
|
||||
RUN version="${VERSION:-$(git describe --tags --always)}"; \
|
||||
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
|
||||
[ "$version" = unknown ]; }; then \
|
||||
echo "no version could be derived although the build context carries .git" >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Gitrev=${VERSION}" \
|
||||
-o /mfer ./cmd/mfer/
|
||||
cd cmd/mfer && \
|
||||
CGO_ENABLED=0 go build -ldflags "-X main.Gitrev=$version" -o /mfer .
|
||||
|
||||
# Fail unless /mfer is statically linked: scratch has no C library to run it.
|
||||
RUN ldd /mfer 2>&1 | grep -q 'not a dynamic executable'
|
||||
|
||||
# Runtime stage, and the last one.
|
||||
FROM scratch
|
||||
# scratch has no CA certificates; fetch needs them to verify HTTPS servers.
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
|
||||
@@ -55,9 +55,9 @@ for a single command's options.
|
||||
|
||||
# Build Status
|
||||
|
||||
CI runs `script/cibuild`, which runs `script/bootstrap` and `script/check`, then
|
||||
builds the Docker image with `--no-cache`, so the lint and test phases in the
|
||||
`Dockerfile` run on every build. The `main` branch must always be green.
|
||||
CI runs `script/cibuild`, which builds the Docker image with `--no-cache`, so
|
||||
the formatting, lint and test steps in the `Dockerfile` run on every build. The
|
||||
`main` branch must always be green.
|
||||
|
||||
# Entrypoints
|
||||
|
||||
@@ -68,22 +68,18 @@ development workflow, and the Makefile targets are thin shims that call them. We
|
||||
provide:
|
||||
|
||||
- `script/bootstrap` — install all dependencies, idempotently: Go and the
|
||||
modules of `go.mod`; node (the version `.nvmrc` names, through nvm when there
|
||||
is no node on `PATH`) and yarn, plus the prettier version pinned in
|
||||
`package.json`/`yarn.lock`; `gofumpt` v0.12.0 and `protoc-gen-go` v1.36.11,
|
||||
installed into `bin/` with `go install`, each pinned to a commit; and `protoc`
|
||||
33.4, unpacked into `bin/protoc` from its release archive once the archive
|
||||
matches the sha256 the script holds for this platform. Each of those three is
|
||||
installed again whenever the one in `bin/` reports another version.
|
||||
golangci-lint is not installed, it runs only in Docker
|
||||
modules of both `go.mod` and `bin/tools/go.mod`; node (the version `.nvmrc`
|
||||
names, through nvm when there is no node on `PATH`) and yarn, plus the
|
||||
prettier version pinned in `package.json`/`yarn.lock`; and `protoc` 33.4,
|
||||
unpacked into `bin/protoc` from its release archive once the archive matches
|
||||
the sha256 the script holds for this platform. golangci-lint is not installed,
|
||||
it runs only in Docker
|
||||
- `script/setup` — make a fresh clone ready for development: runs
|
||||
`script/bootstrap`, then `script/install-precommit`
|
||||
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
||||
such as `script/docker`
|
||||
- `script/test` — run the test suite in Docker: builds only the `test` stage of
|
||||
the `Dockerfile`, whose build runs `go test -race`, uncached so it runs every
|
||||
time; one test fails when `mfer/mf.proto` no longer matches the hash
|
||||
`script/generate` recorded
|
||||
- `script/test` — run the test suite (`go test`); one test fails when
|
||||
`mfer/mf.proto` no longer matches the hash `script/generate` recorded
|
||||
- `script/build` (`make build`) — build the `mfer` binary into `bin/mfer`,
|
||||
stamped with the revision `mfer version` prints: the output of
|
||||
`git describe --tags --always --dirty`, as `script/docker` passes it
|
||||
@@ -91,20 +87,23 @@ provide:
|
||||
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in
|
||||
`mfer/mf.proto.sha256`; the only thing that regenerates the committed
|
||||
`mfer/mf.pb.go`. It runs the `protoc` that `script/bootstrap` unpacks into
|
||||
`bin/protoc` and the `protoc-gen-go` it installs into `bin/`, refusing any
|
||||
version of either but the pinned one
|
||||
`bin/protoc`, refusing any version but 33.4, and the `protoc-gen-go` that
|
||||
`bin/tools/go.mod` pins, which `go tool` builds from source checked against
|
||||
the hashes in `bin/tools/go.sum`
|
||||
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
|
||||
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
|
||||
as ordinary tests
|
||||
- `script/lint` — run `golangci-lint` in Docker: builds only the `lint` stage of
|
||||
the `Dockerfile`, whose build runs the linter, uncached so it runs every time
|
||||
the `Dockerfile` (the Go format check, then the linter), uncached so it runs
|
||||
every time, then removes the image
|
||||
- `script/fmt` — format all code and docs (writes): `script/gofumpt --write` and
|
||||
`script/prettier --write`
|
||||
- `script/gofumpt` — run `gofumpt` over every Go file in the repository in the
|
||||
given mode, `--write` or `--check`, with the `bin/gofumpt` that
|
||||
`script/bootstrap` installs, refusing any version but the pinned one;
|
||||
`script/fmt` and `script/fmt-check` both go through it, so they cannot
|
||||
disagree about Go formatting
|
||||
given mode, `--write` or `--check`, at the version `bin/tools/go.mod` pins
|
||||
(built on demand by `go tool` from source checked against the hashes in
|
||||
`bin/tools/go.sum`, so nothing installs it); `script/fmt`, `script/fmt-check`
|
||||
and the Docker lint stage all go through it, so they cannot disagree about Go
|
||||
formatting
|
||||
- `script/prettier` — run prettier over the repository's canonical file set
|
||||
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
||||
`--check`, with the prettier version `yarn.lock` pins, run by the node on
|
||||
@@ -115,9 +114,8 @@ provide:
|
||||
`script/gofumpt --check` plus `script/prettier --check`
|
||||
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
||||
- `script/docker` — build the Docker image tagged with the project name
|
||||
- `script/cibuild` — CI entrypoint: runs `script/bootstrap` and `script/check`,
|
||||
then builds the image with the same command as `script/docker`, uncached, so
|
||||
the lint and test phases in the `Dockerfile` run every time
|
||||
- `script/cibuild` — CI entrypoint: builds the image with the same command as
|
||||
`script/docker`, uncached, so the checks in the Dockerfile run every time
|
||||
- `script/precommit` — pre-commit checks: `go mod tidy` verification, then
|
||||
`script/check`
|
||||
- `script/install-precommit` — install the git pre-commit hook that runs
|
||||
@@ -261,12 +259,9 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
||||
|
||||
- `mfer gen` / `mfer gen .`
|
||||
- recurses under current directory and writes out an `index.mf`
|
||||
- records every file's mode as `0000` unless given `--include-permissions`,
|
||||
which records each file's permission bits (`0777` at most)
|
||||
- `mfer check` / `mfer check .`
|
||||
- verifies checksums of all files in manifest, displaying error and exiting
|
||||
nonzero if any files are missing or corrupted, or have permission bits
|
||||
other than the mode the manifest records, unless that is `0000`
|
||||
nonzero if any files are missing or corrupted
|
||||
- warns about each file under the base directory that the manifest does not
|
||||
list, hidden files included; with `--no-extra-files` each one is a failure
|
||||
instead
|
||||
@@ -274,18 +269,14 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
||||
- fetches `/stuff/index.mf` and downloads all files listed in manifest into
|
||||
the current directory, or the one given with `--dest`, and assures
|
||||
cryptographic integrity of downloaded files. A file already there with the
|
||||
size, hash and recorded mode the manifest lists is skipped. Once every
|
||||
file is in place, the manifest is saved there as `index.mf`, so
|
||||
`mfer check` can verify the tree later. Each file is downloaded to a temp
|
||||
file beside it, such as `.a.txt.tmp` for `a.txt`, given the mode the
|
||||
manifest records unless that is `0000`, then moved into place. A manifest
|
||||
is refused before any file is downloaded if it records a mode above
|
||||
`0777`, or lists a file where fetch writes another: at another listed file
|
||||
or a directory one is in, at the temp file of a listed file, or at
|
||||
`index.mf` or `.index.mf.tmp` at the top of the tree. Names are compared
|
||||
in any letter case, on every filesystem, since on a case-insensitive one
|
||||
`A.txt` and `a.txt` are one file; a directory two listed files are in must
|
||||
be spelled alike in both.
|
||||
size and hash the manifest lists is skipped. Once every file is in place,
|
||||
the manifest is saved there as `index.mf`, so `mfer check` can verify the
|
||||
tree later. Each file is downloaded to a temp file beside it, such as
|
||||
`.a.txt.tmp` for `a.txt`, then moved into place. A manifest is refused
|
||||
before any file is downloaded if it lists a file where fetch writes
|
||||
another: at the temp file of a listed file, or at `index.mf` or
|
||||
`.index.mf.tmp` at the top of the tree. Names are compared in any letter
|
||||
case.
|
||||
- `mfer fetch --require-signature <fingerprint> https://example.com/stuff/`
|
||||
- as above, but first refuses a manifest not signed by the key with that
|
||||
fingerprint, as `mfer check --require-signature` does, before downloading
|
||||
|
||||
+82
-353
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: Repository Policies
|
||||
last_modified: 2026-10-04
|
||||
last_modified: 2026-07-06
|
||||
---
|
||||
|
||||
This document covers repository structure, tooling, and workflow standards. Code
|
||||
@@ -60,28 +60,17 @@ style conventions are in separate documents:
|
||||
prerequisite since nvm requires bash. yarn is then pinned via
|
||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
|
||||
with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
||||
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
||||
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
||||
pristine checkout with nothing installed the run dies there, after the
|
||||
containerised gates have passed. **The bootstrap alone is not enough**:
|
||||
`script/bootstrap` installs node and yarn under nvm and leaves neither on the
|
||||
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
|
||||
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
|
||||
source nvm for the pinned node version before invoking it, exactly as
|
||||
`script/bootstrap`'s own install step does. A runner carrying nothing but
|
||||
docker and git then gets through `script/check`. Four further scripts are our
|
||||
own extensions to the standard: `script/check` runs `script/test`,
|
||||
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
|
||||
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
|
||||
installs the git pre-commit hook (the `make hooks` target shims to it); and
|
||||
`script/projectname` (literally that filename) simply outputs the project's
|
||||
name. Scripts that need the name call `script/projectname` — e.g.
|
||||
`script/docker` assembles its image tag from it — so those scripts stay
|
||||
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
|
||||
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
|
||||
the hook itself. Model scripts are at
|
||||
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
|
||||
scripts are our own extensions to the standard: `script/check` runs
|
||||
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
|
||||
what the git pre-commit hook runs, and it calls `script/check`;
|
||||
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
|
||||
target shims to it); and `script/projectname` (literally that filename) simply
|
||||
outputs the project's name. Scripts that need the name call
|
||||
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
|
||||
so those scripts stay byte-identical across all repos. Repo-type-specific
|
||||
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
||||
`script/precommit`, not in the hook itself. Model scripts are at
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||
must document the provided scripts in an **Entrypoints** section (see the
|
||||
README requirements below).
|
||||
@@ -100,198 +89,87 @@ style conventions are in separate documents:
|
||||
contributor should be able to understand the entire development workflow by
|
||||
reading the Makefile.
|
||||
|
||||
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a
|
||||
`lint` phase and a `test` phase, with the final stage depending on both so the
|
||||
image cannot be built unless they pass. For non-server repos the final stage
|
||||
brings up a development environment; for server repos it is the runtime image.
|
||||
The gate phases and the build stage start from their pinned base images and
|
||||
install what those images lack either inline, as the canonical Go `Dockerfile`
|
||||
below does for `git`, or by running `script/bootstrap`, as the `prompts`
|
||||
repo's own `Dockerfile` does for its yarn packages. The development
|
||||
environment stage installs development prerequisites by running
|
||||
`script/bootstrap` rather than duplicating its installs inline. A stage that
|
||||
runs `script/bootstrap` COPYs `script/` and the dependency manifests
|
||||
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
|
||||
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
||||
as a build step so the build fails if the branch is not green. For non-server
|
||||
repos, the Dockerfile should bring up a development environment and run
|
||||
`make check`. For server repos, `make check` should run as an early build
|
||||
stage before the final image is assembled. Dockerfiles install development
|
||||
prerequisites by running `script/bootstrap` rather than duplicating installs
|
||||
inline; COPY `script/` and the dependency manifests (`package.json` +
|
||||
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
|
||||
layer stays cached until dependencies change.
|
||||
|
||||
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
|
||||
no separate lint file. `script/lint` and `script/test` each build one phase
|
||||
and nothing else:
|
||||
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
||||
repos use a multistage build where linting runs in an independent stage based
|
||||
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
||||
`make fmt-check` and `make lint` before the full build begins. The build stage
|
||||
then declares an explicit dependency on the lint stage via
|
||||
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
||||
linting before proceeding to compilation and tests. This ensures lint failures
|
||||
surface in seconds rather than minutes, without blocking on dependency
|
||||
download or compilation in the build stage.
|
||||
|
||||
```sh
|
||||
docker build --no-cache --target lint -t "$(script/projectname)-lint" .
|
||||
docker build --no-cache --target test -t "$(script/projectname)-test" .
|
||||
```
|
||||
|
||||
**A stage that is not the last one in the file is built only when the final
|
||||
stage's chain depends on it, or when `--target` names it.** That is why the
|
||||
two gates are always invoked by name here, and why the final stage carries a
|
||||
`COPY --from=` of a harmless file from each of them: without that edge a
|
||||
plain `docker build .` builds the last stage alone and exits 0 having linted
|
||||
and tested nothing.
|
||||
|
||||
**Every `docker build` in `script/` is tagged**, here and in
|
||||
`script/cibuild` and `script/docker`. An untagged build leaves a dangling
|
||||
image behind on every invocation, on every developer host and every CI
|
||||
runner; a tagged one replaces the previous image.
|
||||
|
||||
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
||||
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
||||
themselves a `docker build` and would recurse into a daemon that does not
|
||||
exist in a build step. Formatting is the exception and stays on the host:
|
||||
`script/fmt` writes the working tree, and `script/fmt-check` is its
|
||||
read-only twin.
|
||||
|
||||
**No lint verdict may come from a host invocation of the linter.** On a
|
||||
shared host golangci-lint reads a result cache keyed on file content rather
|
||||
than location, so a second checkout of the same content is served the first
|
||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
||||
org, in both directions. A container has its own cache, its own `TMPDIR` and
|
||||
a digest-pinned binary, so neither is reachable.
|
||||
|
||||
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
|
||||
a `COPY` layer only when the copied content changes, so on an unchanged tree
|
||||
the check `RUN` is served from cache, nothing executes, and the build still
|
||||
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
|
||||
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
|
||||
four, and there is no fifth — `script/check` runs the two gate phases and
|
||||
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
|
||||
not evidence that anything ran: a sub-second build reporting success is a
|
||||
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
|
||||
and friends destroy a build cache shared with every other build on the host.
|
||||
When a check is added or changed, prove it works by planting a defect it must
|
||||
catch and watching the run fail on it, then revert the defect. A green run
|
||||
alone shows neither that the check ran nor that it covers what it should.
|
||||
|
||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
||||
hash), so lint failures surface in seconds rather than after a full compile,
|
||||
and the test phase is based on the Debian Go image. The canonical Go repo
|
||||
`Dockerfile`:
|
||||
The standard pattern for a Go repo Dockerfile is:
|
||||
|
||||
```dockerfile
|
||||
# Lint phase
|
||||
# Lint stage — fast feedback on formatting and lint issues
|
||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||
FROM golangci/golangci-lint@sha256:... AS lint
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
RUN make fmt-check
|
||||
RUN make lint
|
||||
|
||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||
# image ships and the alpine one does not.
|
||||
# golang:1.x, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS test
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go test -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 90s -race -v ./...; exit 1; }
|
||||
|
||||
# Build stage. Nothing is wanted from either phase above; the copies
|
||||
# are what make BuildKit build them first, so this stage cannot run
|
||||
# unless lint and test passed.
|
||||
# Build stage
|
||||
# golang:1.x-alpine, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS builder
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
RUN apk add --no-cache git
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /src
|
||||
WORKDIR /src
|
||||
|
||||
# Force BuildKit to run the lint stage before proceeding
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN make test
|
||||
|
||||
# The VERSION build arg when one is given, otherwise
|
||||
# `git describe --tags --always` on the .git in the build context. With
|
||||
# .git present, a version that is still empty, dev or unknown fails the
|
||||
# build: git is missing or could not read the checkout.
|
||||
ARG VERSION
|
||||
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||
if [ -e .git ]; then \
|
||||
case "$VERSION" in ""|dev|unknown) \
|
||||
echo "version is '$VERSION' although .git is present" >&2; \
|
||||
exit 1 ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /app ./cmd/app/
|
||||
|
||||
# Runtime stage, and the last one
|
||||
# Runtime stage
|
||||
FROM alpine@sha256:...
|
||||
COPY --from=builder /app /usr/local/bin/app
|
||||
ENTRYPOINT ["app"]
|
||||
```
|
||||
|
||||
Key points:
|
||||
- The lint phase uses the `golangci/golangci-lint` image directly (it has
|
||||
both Go and the linter), so nothing needs installing.
|
||||
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only
|
||||
purpose is the ordering edge. BuildKit runs stages in parallel by default,
|
||||
and a stage nothing depends on is not built at all, so without these two
|
||||
lines a red gate would not fail the build.
|
||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
||||
same two copies. A plain `docker build .` builds the last stage's chain
|
||||
and nothing else.
|
||||
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
||||
includes both Go and the linter), so there is no need to install the
|
||||
linter separately.
|
||||
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
||||
a stage dependency. BuildKit runs stages in parallel by default; without
|
||||
this line, the build stage would not wait for lint to finish and a lint
|
||||
failure might not fail the overall build.
|
||||
- If the project uses `//go:embed` directives that reference build artifacts
|
||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
||||
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
||||
create placeholder files so the embed directives resolve. Example:
|
||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||
- If the project requires CGO or system libraries for linting, install them
|
||||
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
||||
has no `apk`, so install with `apt-get` under the Debian package name
|
||||
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
||||
lists in the same `RUN`, so the layer does not keep them:
|
||||
|
||||
```dockerfile
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
```
|
||||
|
||||
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
||||
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
|
||||
repository's own, each submodule's under `.git/modules/`, and that of a
|
||||
submodule keeping its own `.git` directory. `git describe` does not need
|
||||
them, and each can hold a credential: a password in a remote URL, or the
|
||||
token the CI checkout step stores there. A submodule whose name has a
|
||||
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
|
||||
git directory to `**/.git/modules/**/config`, and Go's version stamping
|
||||
then fails the build: give it a name without that segment
|
||||
(`git submodule add --name`). The stage that compiles has `git` (the
|
||||
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
|
||||
takes the version from the `VERSION` build argument when one is given,
|
||||
otherwise from `git describe --tags --always`. That gives the tag on a
|
||||
tagged commit; on a later commit, the tag, the number of commits since it
|
||||
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
|
||||
tag is reachable. The stage that compiles also marks its working directory
|
||||
safe for git (`git config --system --add safe.directory /src`): a context
|
||||
sent as a tar stream keeps the sender's file owners, and git refuses a
|
||||
checkout owned by another user, so the version would come out empty.
|
||||
`ARG VERSION` has no default, and the build fails if the context carries
|
||||
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
|
||||
`docker build .` with no build arguments must succeed; a Dockerfile that
|
||||
refuses an empty build argument drops that refusal and keeps the argument.
|
||||
The lint stage should not depend on the actual build output — it exists to
|
||||
fail fast.
|
||||
- If the project requires CGO or system libraries for linting (e.g.
|
||||
`vips-dev`), install them in the lint stage with `apk add`.
|
||||
- The build stage runs `make test` after compilation setup. Tests run in the
|
||||
build stage, not the lint stage, because they may require compiled
|
||||
artifacts or heavier dependencies.
|
||||
|
||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||
runs `script/cibuild` on push, and checks out the repo as its only other step.
|
||||
That script bootstraps, runs the gate phases, and then builds the image, so a
|
||||
successful run means every check passed; a bare `docker build .` does not
|
||||
carry the same guarantee, because its gate phases may come from the cache. The
|
||||
image build is uncached and so runs the gate phases a second time. That is the
|
||||
price of the rule above, and it is worth paying: the image that ships is built
|
||||
from a run of its own gates rather than from a cache entry. A separate
|
||||
workflow limited to `main` by a `branches` list under `on: push` cannot be
|
||||
checked by review: to try a change to it, add the feature branch to that list
|
||||
and push, then remove the branch from the list again before merging. Keep any
|
||||
job in it that publishes behind `if: github.ref_name == 'main'`, so the run
|
||||
from the feature branch publishes nothing.
|
||||
runs `script/cibuild` (which runs `docker build .`) on push. Since the
|
||||
Dockerfile already runs `make check`, a successful build implies all checks
|
||||
pass.
|
||||
|
||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||
@@ -311,21 +189,14 @@ style conventions are in separate documents:
|
||||
module under test to verify it compiles/parses. There is no excuse for
|
||||
`make test` to be a no-op.
|
||||
|
||||
- `make test` must complete in under 60 seconds. That is the hard cap, and a
|
||||
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
||||
20 and 60 seconds is still green, but the overage must be filed as an
|
||||
improvement bug against that repo. Add a 90-second timeout to the test
|
||||
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
|
||||
hard cap so that it catches a genuinely hung test rather than a merely slow
|
||||
one.
|
||||
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
|
||||
Makefile.
|
||||
|
||||
- **The test command should use the conditional verbose rerun pattern.** Run
|
||||
tests without `-v` (verbose) first. If tests fail, automatically rerun with
|
||||
`-v` to show full output. This keeps CI logs and `docker build` output clean
|
||||
on success (just package/suite summaries) while providing full diagnostic
|
||||
detail on failure (every test case, every assertion). The command lives in the
|
||||
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
|
||||
Makefile form below is the same pattern for any repo-local invocation:
|
||||
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
||||
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
||||
show full output. This keeps CI logs and `docker build` output clean on
|
||||
success (just package/suite summaries) while providing full diagnostic detail
|
||||
on failure (every test case, every assertion). The general shell pattern:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@@ -338,26 +209,11 @@ style conventions are in separate documents:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@go test -count=1 -timeout 90s -race -cover ./... || \
|
||||
@go test -timeout 30s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||
go test -timeout 30s -race -v ./...; exit 1; }
|
||||
```
|
||||
|
||||
`-count=1` is required on both invocations: it defeats Go's test _result_
|
||||
cache, so neither run can report a stored pass in place of running the
|
||||
tests. It leaves the build cache alone, so it costs the runtime of the suite
|
||||
and no recompilation.
|
||||
|
||||
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
||||
passing result in its cache directory (`GOCACHE`), and when the same tests
|
||||
run again on unchanged code it prints that result, marked `(cached)`,
|
||||
without running them. That matters on a developer's machine, where this
|
||||
target runs and the directory lasts from one run to the next. The `test`
|
||||
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
|
||||
result for this repo's tests and nothing before its `go test` step runs a
|
||||
test, so there is nothing to replay. `--no-cache` (above) is what makes that
|
||||
step run on an unchanged tree.
|
||||
|
||||
Python example:
|
||||
|
||||
```makefile
|
||||
@@ -383,84 +239,10 @@ style conventions are in separate documents:
|
||||
must be in `.gitignore`. No exceptions.
|
||||
|
||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
||||
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
|
||||
language build artifacts, and `node_modules/`. Fetch the standard `.gitignore`
|
||||
from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when
|
||||
setting up a new repo. These patterns are written to `.gitignore`'s own
|
||||
semantics, in which an unanchored pattern already matches at every depth; they
|
||||
are not a `.dockerignore` and must not be transplanted into one unmodified.
|
||||
|
||||
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
|
||||
across unmodified leaves secrets in the build context.** Docker matches with
|
||||
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
|
||||
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
|
||||
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
|
||||
therefore excludes only the copies at the repository root, while `config/.env`
|
||||
and `certs/server.key` still reach the context and can land in an image layer
|
||||
— which is more dangerous than a short file with no secret patterns at all,
|
||||
because it reads as solved and stops anyone looking. Give every
|
||||
depth-independent pattern the `**/` prefix and leave only genuinely
|
||||
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
|
||||
binary, written `/myapp` and never `**/myapp`, which would also match
|
||||
`cmd/myapp/` and delete the package directory from the context. Matching is
|
||||
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
|
||||
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
|
||||
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
|
||||
also catches something the build needs, re-include it with a negation
|
||||
(`!docs/example.env`); deleting the pattern reopens the exposure for every
|
||||
other file it covers. Fetch the standard `.dockerignore` from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
|
||||
it with the repo's own artifacts.
|
||||
|
||||
- **In-repo agent scratch belongs in both files, written to each file's own
|
||||
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
|
||||
additional checkout of the repo — so under `COPY . .` the build context
|
||||
inflates by a multiple of the repo and another session's unreviewed work can
|
||||
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
|
||||
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
|
||||
prefix, because the prefixed form would also delete any nested directory of
|
||||
that name from the build. Anchoring carries a known gap that the canonical
|
||||
`.dockerignore` states in its own comment, since consuming repos receive the
|
||||
file and not the tracker: the directory is created in the agent's working
|
||||
directory, so a repo running agents in subdirectories still ships
|
||||
`services/api/.claude/` and must add its own anchored entry there.
|
||||
|
||||
- **A plain `docker build .` of a clone stamps the version that
|
||||
`git describe --tags --always` gives**, derived from the `.git` in the build
|
||||
context as the canonical `Dockerfile` above shows. Without its failure check,
|
||||
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
|
||||
and the build would still exit 0. `script/docker` and `script/cibuild` pass
|
||||
the version they compute on the host; it takes precedence. They do this
|
||||
byte-identically across repos:
|
||||
|
||||
```sh
|
||||
# Own line: a failing command substitution inside an argument does not
|
||||
# trip `set -e`, so the inline form degrades to an empty constant.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
docker build --no-cache \
|
||||
--build-arg VERSION="$version" \
|
||||
-t "$(script/projectname)" .
|
||||
```
|
||||
|
||||
`--always` makes an untagged repo yield an abbreviated commit hash rather
|
||||
than failing, and the `[ -n "$version" ]` line is the single place the
|
||||
fallback is applied — a live check that fires on a build from an export with
|
||||
no `.git` and on a repository with no commits yet. Do not fold it into the
|
||||
substitution as `|| echo unknown`, which makes the guard unreachable. The
|
||||
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
|
||||
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
|
||||
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
|
||||
the scripts stay byte-identical. One consequence for CI: the standard
|
||||
checkout action clones shallow and fetches no tags, so a repo that embeds a
|
||||
tag-derived version must set `fetch-depth: 0` on its checkout step.
|
||||
|
||||
- **Verify `.dockerignore` by enumerating the image, not by reading the
|
||||
patterns.** Plant files at the root _and_ at least two directories deep, build
|
||||
a probe image that does `COPY . .`, and list what actually landed
|
||||
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
|
||||
size is not a substitute: a nested secret is a few bytes, and BuildKit
|
||||
transfers only the delta from the previous build.
|
||||
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
||||
Fetch the standard `.gitignore` from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
||||
a new repo.
|
||||
|
||||
- **No build artifacts in version control.** Code-derived data (compiled
|
||||
bundles, minified output, generated assets) must never be committed to the
|
||||
@@ -476,56 +258,9 @@ style conventions are in separate documents:
|
||||
- Make all changes on a feature branch. You can do whatever you want on a
|
||||
feature branch.
|
||||
|
||||
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must
|
||||
_NEVER_ be modified by an agent: fetch it from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it
|
||||
byte-identical, so that no repo can quietly loosen its own linting. Linter
|
||||
configuration changes are made to the canonical copy in the `prompts` repo and
|
||||
reach consuming repos by re-vendoring; an agent may open a PR against
|
||||
canonical, which only the user merges. One list is exempt from byte-identity,
|
||||
because it cannot be written once for every repo: the `deny` list of the
|
||||
`test-support` depguard rule, where a repo names its own test-support packages
|
||||
by full import path. A repo adds entries there and changes nothing else, and a
|
||||
re-vendor carries its entries forward. The canonical golangci-lint version is
|
||||
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
|
||||
image
|
||||
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
|
||||
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
|
||||
directive must not name a newer Go minor version than the one golangci-lint
|
||||
was built with, or golangci-lint refuses to lint it: this release lints
|
||||
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
|
||||
installs golangci-lint on the host. A repo sets the lint phase digest to the
|
||||
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
|
||||
the two prompted the change: the canonical copy can name linters that an older
|
||||
golangci-lint rejects, and a newer golangci-lint can add linters that
|
||||
`default: all` switches on until the canonical copy disables them.
|
||||
|
||||
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
|
||||
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
|
||||
`PATH` only, so on an already-provisioned machine the pin is inert and a
|
||||
version bump is a silent no-op — while the Dockerfile, installing into a clean
|
||||
image, gets the pinned version, so a local `make check` and `make docker` can
|
||||
disagree about what the tool even is. The canonical form:
|
||||
- compares the installed version against the pin over the **whole** version
|
||||
token; a parser that stops at the first `-` reports `2.12.2` for a host
|
||||
running `2.12.2-rc1` and skips the install;
|
||||
- treats absent, non-zero, empty or unrecognised `--version` output as a
|
||||
mismatch, so the failure direction is a redundant install and never a
|
||||
skipped one;
|
||||
- after installing, re-resolves the binary the way callers do — `hash -r`,
|
||||
then through `PATH`, not through the directory the installer wrote to —
|
||||
and fails naming the resolved path, since an install that a shadowing
|
||||
binary hides succeeds while changing nothing any caller sees;
|
||||
- is actually called, and prints the version on both success paths: a
|
||||
function defined and never invoked has the same exit status and the same
|
||||
empty output as one that worked.
|
||||
|
||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
||||
|
||||
A Go tool a repo needs on the host is installed with `go install` pinned to
|
||||
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
|
||||
a `go.mod` tool dependency or through a `tools.go` file, either of which
|
||||
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
|
||||
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
||||
manually by the user. Fetch from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
|
||||
|
||||
- When pinning images or packages by hash, add a comment above the reference
|
||||
with the version and date (YYYY-MM-DD).
|
||||
@@ -639,14 +374,12 @@ style conventions are in separate documents:
|
||||
settings.
|
||||
|
||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
||||
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`,
|
||||
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`,
|
||||
and language-specific config). Everything else goes in a subdirectory.
|
||||
Canonical subdirectory names:
|
||||
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
||||
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
||||
language-specific config). Everything else goes in a subdirectory. Canonical
|
||||
subdirectory names:
|
||||
- `bin/` — executable scripts and tools
|
||||
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
|
||||
body is a single call into `internal/` or `pkg/`, no project logic in
|
||||
`cmd/`
|
||||
- `cmd/` — Go command entrypoints
|
||||
- `configs/` — configuration templates and examples
|
||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
||||
- `docs/` — documentation and markdown (README.md stays in root)
|
||||
@@ -673,7 +406,3 @@ style conventions are in separate documents:
|
||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
||||
- Python: `pyproject.toml`
|
||||
|
||||
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
|
||||
is never committed under a file or directory named after one agent tool, such
|
||||
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// The developer tools this repo runs with `go tool`: gofumpt for
|
||||
// script/gofumpt and protoc-gen-go for script/generate. Kept out of the mfer
|
||||
// module so they add nothing to what mfer's users download. `go tool` builds
|
||||
// exactly the source whose hashes go.sum here records.
|
||||
module sneak.berlin/go/mfer/bin/tools
|
||||
|
||||
go 1.26.0
|
||||
|
||||
tool (
|
||||
google.golang.org/protobuf/cmd/protoc-gen-go
|
||||
mvdan.cc/gofumpt
|
||||
)
|
||||
|
||||
require (
|
||||
golang.org/x/mod v0.40.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/tools v0.49.0 // indirect
|
||||
// protoc-gen-go v1.36.11, 2026-10-04
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
// gofumpt v0.12.0, 2026-10-04
|
||||
mvdan.cc/gofumpt v0.12.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
|
||||
github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
|
||||
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
mvdan.cc/gofumpt v0.12.0 h1:1Lbudkz2kpM9Cjz2pL4M19u7q+GaEhCTNf7N9mfpcho=
|
||||
mvdan.cc/gofumpt v0.12.0/go.mod h1:SmBHHrljiZu/uoypeKup3rFzP6eoC9UwCp2iH5E3jZA=
|
||||
+6
-21
@@ -6,11 +6,8 @@ Version 1.0
|
||||
|
||||
An `.mf` file is a binary manifest that describes a directory tree of files,
|
||||
including their paths, sizes, and cryptographic checksums. It supports optional
|
||||
GPG signatures for integrity verification and optional timestamps and file
|
||||
permissions for metadata preservation.
|
||||
|
||||
Nothing goes in the 1.0 manifest that 1.0 does not read or write: no field is
|
||||
reserved or kept for later use.
|
||||
GPG signatures for integrity verification and optional timestamps for metadata
|
||||
preservation.
|
||||
|
||||
## File Structure
|
||||
|
||||
@@ -53,7 +50,7 @@ enforce a decompression size limit to prevent decompression bombs. The reference
|
||||
implementation limits decompressed size to 256 MB. It writes zstd frames with a
|
||||
window of at most 8 MiB, the largest window the zstd format recommends decoders
|
||||
support, and refuses frames that ask for a larger one. It also refuses an inner
|
||||
message whose file entries, hashes, timestamps and MIME types, counted at 176,
|
||||
message whose file entries, hashes, timestamps and MIME types, counted at 160,
|
||||
112, 64 and 16 bytes each, add up to more than 8 times its size.
|
||||
|
||||
## Inner Message (`MFFile`)
|
||||
@@ -80,21 +77,9 @@ Each file entry contains:
|
||||
| `mimeType` | 301 | string (optional) | MIME type |
|
||||
| `mtime` | 302 | Timestamp (optional) | Modification time |
|
||||
| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) |
|
||||
| `mode` | 304 | uint32 | Permission bits (see File Mode) |
|
||||
|
||||
## File Mode
|
||||
|
||||
`mode` holds a file's Unix permission bits, the nine `rwx` bits, so it is never
|
||||
above `0777` (octal); the setuid, setgid and sticky bits are never recorded.
|
||||
Writers record `0000` unless whoever creates the manifest asks for permissions.
|
||||
`0000`, the proto3 default, means no mode was recorded: readers never check or
|
||||
apply it.
|
||||
|
||||
The reference implementation records modes when `gen` or `freshen` is given
|
||||
`--include-permissions`. `check` fails a file whose permission bits differ from
|
||||
a recorded mode other than `0000`. `fetch` sets a recorded mode other than
|
||||
`0000` on each file it writes, and refuses a manifest that records a mode above
|
||||
`0777` before it requests any file.
|
||||
Field 304 (`atime`) has been removed from the specification. Access time is
|
||||
volatile and non-deterministic; it is not useful for integrity verification.
|
||||
|
||||
## Path Rules
|
||||
|
||||
@@ -145,7 +130,7 @@ By default, manifests are generated deterministically:
|
||||
|
||||
- File entries are sorted by `path` in **lexicographic byte order**
|
||||
- `createdAt` is omitted unless explicitly requested
|
||||
- `mode` is `0000` unless explicitly requested
|
||||
- `atime` is never included (field removed from schema)
|
||||
|
||||
This ensures that two independent runs over the same directory tree produce
|
||||
byte-identical `.mf` files (assuming file contents and metadata have not
|
||||
|
||||
@@ -3,7 +3,6 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -375,98 +374,6 @@ func TestGenerateAndCheckCommand(t *testing.T) {
|
||||
assert.Equal(t, 0, exitCode, "check failed: %s", testStderr(t, opts))
|
||||
}
|
||||
|
||||
// TestGenerateRecordsModeOnlyWhenAsked runs gen with and without
|
||||
// --include-permissions: without it every mode is recorded as 0000, with
|
||||
// it each file's permission bits. list -l and export show the recorded
|
||||
// modes.
|
||||
func TestGenerateRecordsModeOnlyWhenAsked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, "/testdir/notes.txt", "hello world")
|
||||
writeTestFile(t, fs, "/testdir/run.sh", "#!/bin/sh\n")
|
||||
require.NoError(t, fs.Chmod("/testdir/run.sh", 0o755))
|
||||
|
||||
for _, tc := range []struct {
|
||||
flags []string
|
||||
want map[string]string // recorded mode by path
|
||||
}{
|
||||
{nil, map[string]string{"notes.txt": "0000", "run.sh": "0000"}},
|
||||
{
|
||||
[]string{"--" + flagIncludePermissions},
|
||||
map[string]string{"notes.txt": "0644", "run.sh": "0755"},
|
||||
},
|
||||
} {
|
||||
opts := testOpts(slices.Concat(
|
||||
[]string{testApp, cmdGenerate, "-q", "-f", "-o", testOutput}, tc.flags,
|
||||
[]string{testDir},
|
||||
), fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
opts = testOpts([]string{testApp, cmdList, "-l", testOutput}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
listed := map[string]string{}
|
||||
out := strings.TrimSuffix(testStdout(t, opts), "\n")
|
||||
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
fields := strings.Split(line, "\t") // mode, size, mtime, path
|
||||
require.Len(t, fields, 4, line)
|
||||
listed[fields[3]] = fields[0]
|
||||
}
|
||||
|
||||
assert.Equal(t, tc.want, listed, "list -l %v", tc.flags)
|
||||
|
||||
opts = testOpts([]string{testApp, cmdExport, testOutput}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
var entries []ExportEntry
|
||||
require.NoError(t, json.Unmarshal([]byte(testStdout(t, opts)), &entries))
|
||||
|
||||
exported := map[string]string{}
|
||||
for _, e := range entries {
|
||||
exported[e.Path] = e.Mode
|
||||
}
|
||||
|
||||
assert.Equal(t, tc.want, exported, "export %v", tc.flags)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckComparesRecordedMode changes a file's mode after gen: check
|
||||
// must fail on it when gen recorded the file's mode, and pass when gen
|
||||
// recorded 0000.
|
||||
func TestCheckComparesRecordedMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
flags []string
|
||||
exitCode int
|
||||
}{
|
||||
{nil, 0},
|
||||
{[]string{"--" + flagIncludePermissions}, 1},
|
||||
} {
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, testFile1, "hello world")
|
||||
|
||||
opts := testOpts(slices.Concat(
|
||||
[]string{testApp, cmdGenerate, "-q", "-o", testMF}, tc.flags,
|
||||
[]string{testDir},
|
||||
), fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
require.NoError(t, fs.Chmod(testFile1, 0o600))
|
||||
|
||||
opts = testOpts([]string{testApp, cmdCheck, testFlagBase, testDir, testMF}, fs)
|
||||
assert.Equal(t, tc.exitCode, runCLI(opts), "%v: %s", tc.flags, testStderr(t, opts))
|
||||
|
||||
if tc.exitCode != 0 {
|
||||
assert.Contains(t, testStderr(t, opts), "MODE_MISMATCH: file1.txt")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sharedWriter appends to a buffer shared with other sharedWriters, so
|
||||
// output written to stdout and stderr is kept in the order it was written.
|
||||
// Each write first waits for delay.
|
||||
|
||||
@@ -162,7 +162,7 @@ func signedManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
|
||||
for path, content := range files {
|
||||
_, err = b.AddFile(mfer.RelFilePath(path), mfer.FileSize(len(content)),
|
||||
mfer.ModTime{}, 0, bytes.NewReader(content), nil)
|
||||
mfer.ModTime{}, bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ type ExportEntry struct {
|
||||
Hashes []string `json:"hashes"`
|
||||
Mtime *string `json:"mtime,omitempty"`
|
||||
Ctime *string `json:"ctime,omitempty"`
|
||||
Mode string `json:"mode"` // octal, "0000" when none was recorded
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) exportManifestOperation(
|
||||
@@ -50,7 +49,6 @@ func (mfa *CLIApp) exportManifestOperation(
|
||||
Path: f.GetPath(),
|
||||
Size: f.GetSize(),
|
||||
Hashes: make([]string, 0, len(f.GetHashes())),
|
||||
Mode: fmt.Sprintf("%04o", f.GetMode()),
|
||||
}
|
||||
|
||||
for _, h := range f.GetHashes() {
|
||||
|
||||
@@ -132,7 +132,7 @@ func TestListFromHTTPURL(t *testing.T) {
|
||||
|
||||
exitCode := runCLI(&RunOptions{
|
||||
Appname: testApp,
|
||||
Args: []string{testApp, cmdList, server.URL + "/index.mf"},
|
||||
Args: []string{testApp, "list", server.URL + "/index.mf"},
|
||||
Stdin: &bytes.Buffer{},
|
||||
Stdout: &stdout,
|
||||
Stderr: &stderr,
|
||||
|
||||
+20
-65
@@ -95,9 +95,6 @@ var (
|
||||
// writes another file.
|
||||
errNameClash = errors.New(
|
||||
"manifest lists a file where fetch writes another file")
|
||||
// errModeOutOfRange indicates a manifest that lists a mode with more
|
||||
// than the permission bits, such as setuid.
|
||||
errModeOutOfRange = errors.New("manifest lists a mode outside 0777")
|
||||
)
|
||||
|
||||
// DownloadProgress reports the progress of a single file download.
|
||||
@@ -295,11 +292,11 @@ func downloadManifestFiles(
|
||||
}
|
||||
|
||||
// alreadyPresent reports whether localPath under dest is a regular file
|
||||
// with the size, the recorded mode if any, and one of the hashes the
|
||||
// manifest lists for entry. It hashes the whole file, since a matching
|
||||
// size alone would accept a corrupted or partly written one. A file it
|
||||
// cannot read, or reaches only through a symlink, is not present: fetch
|
||||
// downloads it, and the download reports the problem.
|
||||
// with the size and one of the hashes the manifest lists for entry. It
|
||||
// hashes the whole file, since a matching size alone would accept a
|
||||
// corrupted or partly written one. A file it cannot read, or reaches only
|
||||
// through a symlink, is not present: fetch downloads it, and the download
|
||||
// reports the problem.
|
||||
func alreadyPresent(dest, localPath string, entry *mfer.MFFilePath) bool {
|
||||
if checkNoSymlinks(dest, localPath) != nil {
|
||||
return false
|
||||
@@ -312,12 +309,6 @@ func alreadyPresent(dest, localPath string, entry *mfer.MFFilePath) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// A recorded mode of 0 means none was recorded.
|
||||
if entry.GetMode() != 0 &&
|
||||
info.Mode().Perm() != os.FileMode(entry.GetMode()).Perm() {
|
||||
return false
|
||||
}
|
||||
|
||||
// G304: localPath is a relative path that sanitizePath keeps inside
|
||||
// dest as text, and checkNoSymlinks just found no symlink in it.
|
||||
f, err := os.Open(path) //nolint:gosec // G304: see comment above
|
||||
@@ -423,9 +414,9 @@ func (mfa *CLIApp) fetchManifestOperation(
|
||||
|
||||
// fetchManifest downloads the manifest at manifestURL and parses it,
|
||||
// enforcing --require-signature if it is given and refusing a manifest
|
||||
// that lists a file where fetch writes another or a mode outside 0777. It
|
||||
// returns the manifest as downloaded, to be saved once the files are in
|
||||
// place, and the files it lists.
|
||||
// that lists a file where fetch writes another. It returns the manifest as
|
||||
// downloaded, to be saved once the files are in place, and the files it
|
||||
// lists.
|
||||
func fetchManifest(
|
||||
ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
|
||||
) ([]byte, []*mfer.MFFilePath, error) {
|
||||
@@ -469,34 +460,23 @@ func fetchManifest(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// fetch sets each recorded mode on the file it writes, so a mode above
|
||||
// 0777, which could carry setuid, setgid or sticky bits, is refused
|
||||
// before any file is requested.
|
||||
for _, f := range files {
|
||||
if f.GetMode() > uint32(os.ModePerm) {
|
||||
return nil, nil, fmt.Errorf("%w: %s (%#o)",
|
||||
errModeOutOfRange, f.GetPath(), f.GetMode())
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("manifest contains %d files", len(files))
|
||||
|
||||
return manifestData, files, nil
|
||||
}
|
||||
|
||||
// checkNoNameClash returns an error if files lists a file, or a directory
|
||||
// a file is in, under a name where fetch writes another file: another
|
||||
// listed file, a directory another listed file is in, the temp file it
|
||||
// downloads a listed file to, or, at the top of the tree, the saved
|
||||
// manifest or its temp file. fetch would remove or replace what is
|
||||
// a file is in, under a name where fetch writes another file: the temp
|
||||
// file it downloads a listed file to, or, at the top of the tree, the
|
||||
// saved manifest or its temp file. fetch would remove or replace what is
|
||||
// listed there, or fail partway, leaving a tree check rejects. Names are
|
||||
// compared ignoring case, on every filesystem, since on a
|
||||
// case-insensitive one A.txt and a.txt are one file.
|
||||
// compared ignoring case, since on a case-insensitive filesystem INDEX.MF
|
||||
// and index.mf are one file.
|
||||
func checkNoNameClash(files []*mfer.MFFilePath) error {
|
||||
sep := string(filepath.Separator)
|
||||
|
||||
// written maps each name fetch writes, in lower case, to the file or
|
||||
// directory it writes there.
|
||||
// written maps each name fetch writes, other than the listed files
|
||||
// themselves, in lower case, to the file it writes there.
|
||||
written := map[string]string{
|
||||
defaultManifestName: "the saved manifest",
|
||||
tempPathFor(defaultManifestName): "the saved manifest's temp file",
|
||||
@@ -508,26 +488,14 @@ func checkNoNameClash(files []*mfer.MFFilePath) error {
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
// Look up and add each directory on the file's path, then the
|
||||
// file itself. Only the same directory, spelled alike, may
|
||||
// already be there.
|
||||
parts := strings.Split(filepath.Clean(f.GetPath()), sep)
|
||||
last := len(parts) - 1
|
||||
// Look up each directory on the file's path, then the file itself.
|
||||
parts := strings.Split(strings.ToLower(filepath.Clean(f.GetPath())), sep)
|
||||
|
||||
for i := range parts {
|
||||
name := strings.Join(parts[:i+1], sep)
|
||||
|
||||
what := "the directory " + name
|
||||
if i == last {
|
||||
what = "the file " + f.GetPath()
|
||||
what, ok := written[strings.Join(parts[:i+1], sep)]
|
||||
if ok {
|
||||
return fmt.Errorf("%w: %s (%s)", errNameClash, f.GetPath(), what)
|
||||
}
|
||||
|
||||
other, ok := written[strings.ToLower(name)]
|
||||
if ok && (i == last || other != what) {
|
||||
return fmt.Errorf("%w: %s (%s)", errNameClash, f.GetPath(), other)
|
||||
}
|
||||
|
||||
written[strings.ToLower(name)] = what
|
||||
}
|
||||
}
|
||||
|
||||
@@ -885,19 +853,6 @@ func saveResponse(
|
||||
return err
|
||||
}
|
||||
|
||||
// A recorded mode of 0 means none was recorded. Of any other, only the
|
||||
// permission bits are set, whatever the umask, so setuid, setgid and
|
||||
// sticky never are, whichever caller passed the entry.
|
||||
if entry.GetMode() != 0 {
|
||||
err = out.Chmod(os.FileMode(entry.GetMode()).Perm())
|
||||
if err != nil {
|
||||
_ = out.Close()
|
||||
_ = os.Remove(filepath.Join(dest, tmpPath))
|
||||
|
||||
return fmt.Errorf("failed to set mode: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set up hash computation
|
||||
h := sha256.New()
|
||||
|
||||
|
||||
+1
-256
@@ -4,7 +4,6 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
@@ -20,13 +19,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
@@ -1241,256 +1236,6 @@ func TestFetchRefusesListedTempName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchRefusesNamesEqualIgnoringCase fetches manifests that list two
|
||||
// paths that are one name on a case-insensitive filesystem. Fetched
|
||||
// there, of two such files, at the top of the tree or in a directory,
|
||||
// one replaces the other and fetch exits 0. A file and a directory
|
||||
// another file is in stop fetch partway with a non-zero exit, leaving a
|
||||
// partial tree. Two spellings of one directory put both files in one
|
||||
// directory, one of them under a spelling the manifest does not list,
|
||||
// and check reports that file as not in the manifest. So fetch must
|
||||
// refuse each on every filesystem before it creates the destination or
|
||||
// requests any file. A file and a directory with the same name, and a
|
||||
// file listed twice, are refused the same way. A manifest whose names
|
||||
// differ in more than letter case is fetched in full.
|
||||
func TestFetchRefusesNamesEqualIgnoringCase(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct{ first, second, message string }{
|
||||
{"X.txt", "x.txt", "x.txt (the file X.txt)"},
|
||||
{"sub/X.txt", "sub/x.txt", "sub/x.txt (the file sub/X.txt)"},
|
||||
{"Dir", "dir/x", "dir/x (the file Dir)"},
|
||||
{"Dir/a.txt", "dir/b.txt", "dir/b.txt (the directory Dir)"},
|
||||
{"dir", "dir/x", "dir/x (the file dir)"},
|
||||
{"./x.txt", "x.txt", "x.txt (the file ./x.txt)"},
|
||||
} {
|
||||
t.Run(tc.first+" and "+tc.second, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
files := map[string][]byte{
|
||||
tc.first: []byte("the first file"),
|
||||
tc.second: []byte("the second file"),
|
||||
}
|
||||
|
||||
assertFetchRefused(t, builtManifest(t, files), files,
|
||||
"manifest lists a file where fetch writes another file: "+tc.message)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("names that differ in more than letter case", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
files := map[string][]byte{
|
||||
"A.txt": []byte("at the top"),
|
||||
"B.txt": []byte("also at the top"),
|
||||
"dir/a.txt": []byte("in a directory"),
|
||||
"dir/b.txt": []byte("in the same directory"),
|
||||
}
|
||||
manifest := builtManifest(t, files)
|
||||
|
||||
server := httptest.NewServer(fetchTestHandler(manifest, files))
|
||||
defer server.Close()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL,
|
||||
}, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
want := maps.Clone(files)
|
||||
want[defaultManifestName] = manifest
|
||||
assert.Equal(t, want, filesUnder(t, dest))
|
||||
})
|
||||
}
|
||||
|
||||
// TestFetchSetsRecordedMode fetches a tree whose manifest records the
|
||||
// modes 0640 and 0755. Each file must get its mode whatever the umask, and
|
||||
// check must pass on the result. After one file's mode is changed, a
|
||||
// second fetch must download that file again, and only it, to restore its
|
||||
// mode.
|
||||
func TestFetchSetsRecordedMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
files := map[string][]byte{
|
||||
testFileTxt: []byte("a file"),
|
||||
"tool.sh": []byte("#!/bin/sh\n"),
|
||||
}
|
||||
modes := map[string]os.FileMode{testFileTxt: 0o640, "tool.sh": 0o755}
|
||||
|
||||
sourceFs := afero.NewMemMapFs()
|
||||
for p, content := range files {
|
||||
require.NoError(t, afero.WriteFile(sourceFs, "/"+p, content, modes[p]))
|
||||
}
|
||||
|
||||
scanner := mfer.NewScannerWithOptions(&mfer.ScannerOptions{
|
||||
Fs: sourceFs,
|
||||
IncludePermissions: true,
|
||||
})
|
||||
require.NoError(t, scanner.EnumerateFS(sourceFs, "/", nil))
|
||||
|
||||
var manifest bytes.Buffer
|
||||
require.NoError(t, scanner.ToManifest(context.Background(), &manifest, nil))
|
||||
|
||||
tree := fetchTestHandler(manifest.Bytes(), files)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
requested []string
|
||||
)
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
|
||||
requested = append(requested, r.URL.Path)
|
||||
|
||||
mu.Unlock()
|
||||
|
||||
tree.ServeHTTP(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
dest := t.TempDir()
|
||||
fetch := []string{testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL}
|
||||
|
||||
opts := testOpts(fetch, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
for p, mode := range modes {
|
||||
info, err := os.Stat(filepath.Join(dest, p))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, mode, info.Mode().Perm(), p)
|
||||
}
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, cmdCheck, "-q", testFlagBase, dest,
|
||||
filepath.Join(dest, defaultManifestName),
|
||||
}, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
require.NoError(t, os.Chmod(filepath.Join(dest, testFileTxt), 0o600))
|
||||
|
||||
mu.Lock()
|
||||
requested = nil
|
||||
mu.Unlock()
|
||||
|
||||
opts = testOpts(fetch, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
info, err := os.Stat(filepath.Join(dest, testFileTxt))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o640), info.Mode().Perm())
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
assert.ElementsMatch(t,
|
||||
[]string{"/" + defaultManifestName, "/" + testFileTxt}, requested)
|
||||
}
|
||||
|
||||
// TestFetchRefusesModeOutsidePermissionBits fetches manifests that record
|
||||
// a mode with the setuid bit, once as Unix writes it (04755) and once as
|
||||
// Go keeps it in os.FileMode. fetch must refuse both before it creates the
|
||||
// destination or requests any file.
|
||||
func TestFetchRefusesModeOutsidePermissionBits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
files := map[string][]byte{testFileTxt: []byte("a file")}
|
||||
|
||||
assertFetchRefused(t, manifestWithMode(t, testFileTxt, files[testFileTxt], 0o4755),
|
||||
files, "manifest lists a mode outside 0777: file.txt (04755)")
|
||||
|
||||
assertFetchRefused(t,
|
||||
manifestWithMode(t, testFileTxt, files[testFileTxt],
|
||||
uint32(os.ModeSetuid|0o755)),
|
||||
files, "manifest lists a mode outside 0777: file.txt (040000755)")
|
||||
}
|
||||
|
||||
// TestDownloadFileSetsOnlyPermissionBits downloads a file whose entry
|
||||
// records mode 0755 together with setuid, setgid or sticky, as Go keeps
|
||||
// them in os.FileMode. fetchManifest would refuse such an entry; called
|
||||
// directly, downloadFile must still set only the permission bits.
|
||||
func TestDownloadFileSetsOnlyPermissionBits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
content := []byte("#!/bin/sh\n")
|
||||
|
||||
digest := sha256.Sum256(content)
|
||||
hash, err := multihash.Encode(digest[:], multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(content)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
for _, special := range []os.FileMode{os.ModeSetuid, os.ModeSetgid, os.ModeSticky} {
|
||||
entry := &mfer.MFFilePath{
|
||||
Path: testFileTxt,
|
||||
Size: int64(len(content)),
|
||||
Hashes: []*mfer.MFFileChecksum{{MultiHash: hash}},
|
||||
Mode: uint32(special | 0o755),
|
||||
}
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
err := downloadFile(context.Background(), testClient(),
|
||||
server.URL+"/"+testFileTxt, dest, testFileTxt, entry, nil)
|
||||
require.NoError(t, err, special)
|
||||
|
||||
info, err := os.Stat(filepath.Join(dest, testFileTxt))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode(), special)
|
||||
}
|
||||
}
|
||||
|
||||
// manifestWithMode returns a manifest listing one file, path, with content
|
||||
// and the given recorded mode. It is assembled by hand, since the builder
|
||||
// never records a mode outside 0777.
|
||||
func manifestWithMode(t *testing.T, path string, content []byte, mode uint32) []byte {
|
||||
t.Helper()
|
||||
|
||||
digest := sha256.Sum256(content)
|
||||
hash, err := multihash.Encode(digest[:], multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
id := uuid.New()
|
||||
|
||||
inner, err := proto.Marshal(&mfer.MFFile{
|
||||
Version: mfer.MFFile_VERSION_ONE,
|
||||
Files: []*mfer.MFFilePath{{
|
||||
Path: path,
|
||||
Size: int64(len(content)),
|
||||
Hashes: []*mfer.MFFileChecksum{{MultiHash: hash}},
|
||||
Mode: mode,
|
||||
}},
|
||||
Uuid: id[:],
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
encoder, err := zstd.NewWriter(nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
compressed := encoder.EncodeAll(inner, nil)
|
||||
require.NoError(t, encoder.Close())
|
||||
|
||||
sum := sha256.Sum256(compressed)
|
||||
|
||||
outer, err := proto.Marshal(&mfer.MFFileOuter{
|
||||
Version: mfer.MFFileOuter_VERSION_ONE,
|
||||
CompressionType: mfer.MFFileOuter_COMPRESSION_ZSTD,
|
||||
Size: int64(len(inner)),
|
||||
Sha256: sum[:],
|
||||
Uuid: id[:],
|
||||
InnerMessage: compressed,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return append([]byte(mfer.MAGIC), outer...)
|
||||
}
|
||||
|
||||
// builtManifest returns a manifest of files, built directly rather than
|
||||
// scanned, since a scan lists no hidden files and never a path starting
|
||||
// with "./".
|
||||
@@ -1501,7 +1246,7 @@ func builtManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
|
||||
for p, content := range files {
|
||||
_, err := builder.AddFile(mfer.RelFilePath(p), mfer.FileSize(len(content)),
|
||||
mfer.ModTime(time.Now()), 0, bytes.NewReader(content), nil)
|
||||
mfer.ModTime(time.Now()), bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
|
||||
+9
-24
@@ -48,7 +48,6 @@ type freshenEntry struct {
|
||||
path string
|
||||
size int64
|
||||
mtime time.Time
|
||||
mode fs.FileMode // mode to record, 0 for none
|
||||
needsHash bool // true if new or changed
|
||||
existing *mfer.MFFilePath // existing manifest entry if unchanged
|
||||
}
|
||||
@@ -61,7 +60,6 @@ type freshenScanner struct {
|
||||
excluded []fs.FileInfo // files left out of the listing
|
||||
includeDotfiles bool
|
||||
followSymlinks bool
|
||||
includePermissions bool
|
||||
showProgress bool
|
||||
existingByPath map[string]*mfer.MFFilePath
|
||||
|
||||
@@ -95,12 +93,6 @@ func (s *freshenScanner) resolveSymlink(path string) (fs.FileInfo, bool) {
|
||||
// recordEntry classifies a scanned file as changed, unchanged, or added
|
||||
// relative to the existing manifest.
|
||||
func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
// A mode of 0 records 0000, which means none was recorded.
|
||||
var mode fs.FileMode
|
||||
if s.includePermissions {
|
||||
mode = info.Mode().Perm()
|
||||
}
|
||||
|
||||
existing, inManifest := s.existingByPath[relPath]
|
||||
if !inManifest {
|
||||
s.added++
|
||||
@@ -110,17 +102,16 @@ func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
mode: mode,
|
||||
needsHash: true,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Check if changed (size, mtime, or the mode to record). An entry
|
||||
// with no recorded mtime cannot be compared, so it counts as changed
|
||||
// and gets re-hashed; silently treating the absent mtime as the Unix
|
||||
// epoch would classify every such entry as changed without saying why.
|
||||
// Check if changed (size or mtime). An entry with no recorded mtime
|
||||
// cannot be compared, so it counts as changed and gets re-hashed;
|
||||
// silently treating the absent mtime as the Unix epoch would classify
|
||||
// every such entry as changed without saying why.
|
||||
existingMtime, haveMtime := entryMtime(existing)
|
||||
if !haveMtime {
|
||||
log.Debugf("%s: manifest entry has no mtime, treating as changed",
|
||||
@@ -128,8 +119,7 @@ func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
}
|
||||
|
||||
if !haveMtime || existing.GetSize() != info.Size() ||
|
||||
!existingMtime.Equal(info.ModTime()) ||
|
||||
fs.FileMode(existing.GetMode()) != mode {
|
||||
!existingMtime.Equal(info.ModTime()) {
|
||||
s.changed++
|
||||
|
||||
log.Verbosef("M %s", relPath)
|
||||
@@ -137,7 +127,6 @@ func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
mode: mode,
|
||||
needsHash: true,
|
||||
})
|
||||
} else {
|
||||
@@ -147,7 +136,6 @@ func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
mode: mode,
|
||||
needsHash: false,
|
||||
existing: existing,
|
||||
})
|
||||
@@ -308,7 +296,7 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
||||
h.hashedFiles++
|
||||
|
||||
// Add to builder with computed hash
|
||||
err = addFileToBuilder(h.builder, e.path, e.size, e.mtime, e.mode, hash)
|
||||
err = addFileToBuilder(h.builder, e.path, e.size, e.mtime, hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
||||
}
|
||||
@@ -396,7 +384,6 @@ func (mfa *CLIApp) freshenScan(
|
||||
excluded: excluded,
|
||||
includeDotfiles: cmd.Bool("include-dotfiles"),
|
||||
followSymlinks: cmd.Bool("follow-symlinks"),
|
||||
includePermissions: cmd.Bool(flagIncludePermissions),
|
||||
showProgress: showProgress,
|
||||
existingByPath: existingByPath,
|
||||
}
|
||||
@@ -624,12 +611,10 @@ func hashFile(r io.Reader, progress func(int64)) ([]byte, int64, error) {
|
||||
|
||||
// addFileToBuilder adds a new file entry to the builder
|
||||
func addFileToBuilder(
|
||||
b *mfer.Builder, path string, size int64, mtime time.Time, mode fs.FileMode,
|
||||
hash []byte,
|
||||
b *mfer.Builder, path string, size int64, mtime time.Time, hash []byte,
|
||||
) error {
|
||||
return b.AddFileWithHash(
|
||||
mfer.RelFilePath(path), mfer.FileSize(size), mfer.ModTime(mtime), mode,
|
||||
hash)
|
||||
mfer.RelFilePath(path), mfer.FileSize(size), mfer.ModTime(mtime), hash)
|
||||
}
|
||||
|
||||
// addExistingToBuilder adds an existing manifest entry to the builder.
|
||||
@@ -649,7 +634,7 @@ func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
||||
|
||||
err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
||||
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
||||
fs.FileMode(entry.GetMode()), entry.GetHashes()[0].GetMultiHash())
|
||||
entry.GetHashes()[0].GetMultiHash())
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"manifest entry %s: %w (regenerate the manifest with mfer generate)",
|
||||
|
||||
@@ -99,42 +99,6 @@ func TestFreshenUnchanged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFreshenRecordsModeOnlyWhenAsked changes only the modes of a tree
|
||||
// after gen made its manifest, which recorded every mode as 0000. freshen
|
||||
// --include-permissions must record each file's permission bits, and a
|
||||
// later freshen without it must record 0000 again, although no file's
|
||||
// content or mtime changed.
|
||||
func TestFreshenRecordsModeOnlyWhenAsked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
root, manifestPath := setupFreshenDir(t, fs,
|
||||
map[string]string{testFileTxt: "a file", testDirFile: "a file in dir"})
|
||||
require.NoError(t, fs.Chmod(filepath.Join(root, testFileTxt), 0o640))
|
||||
require.NoError(t, fs.Chmod(filepath.Join(root, testDirFile), 0o755))
|
||||
|
||||
recordedModes := func() map[string]uint32 {
|
||||
modes := map[string]uint32{}
|
||||
for _, f := range manifestFiles(t, fs, manifestPath) {
|
||||
modes[f.GetPath()] = f.GetMode()
|
||||
}
|
||||
|
||||
return modes
|
||||
}
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFreshen, "-q", "--" + flagIncludePermissions,
|
||||
testFlagBase, root, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
assert.Equal(t, map[string]uint32{testFileTxt: 0o640, testDirFile: 0o755},
|
||||
recordedModes())
|
||||
|
||||
runFreshen(t, fs, root, manifestPath)
|
||||
assert.Equal(t, map[string]uint32{testFileTxt: 0, testDirFile: 0},
|
||||
recordedModes())
|
||||
}
|
||||
|
||||
// assertManifestLists asserts that the manifest at manifestPath lists
|
||||
// exactly the files in want, each with the size and SHA-256 hash of its
|
||||
// content in want and the mtime of the file of that name under root.
|
||||
|
||||
@@ -95,7 +95,6 @@ func (mfa *CLIApp) buildScannerOptions(cmd *cli.Command) *mfer.ScannerOptions {
|
||||
IncludeDotfiles: cmd.Bool("include-dotfiles"),
|
||||
FollowSymLinks: cmd.Bool("follow-symlinks"),
|
||||
IncludeTimestamps: cmd.Bool("include-timestamps"),
|
||||
IncludePermissions: cmd.Bool(flagIncludePermissions),
|
||||
Fs: mfa.Fs,
|
||||
// Neither a manifest being replaced nor a temp file left by an
|
||||
// interrupted run belongs in the new manifest.
|
||||
|
||||
@@ -52,8 +52,8 @@ func (mfa *CLIApp) listManifestOperation(ctx context.Context, cmd *cli.Command)
|
||||
mtimeStr = mtime.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(mfa.Stdout, "%04o\t%d\t%s\t%s%s",
|
||||
f.GetMode(), f.GetSize(), mtimeStr, f.GetPath(), lineEnd)
|
||||
_, _ = fmt.Fprintf(mfa.Stdout, "%d\t%s\t%s%s",
|
||||
f.GetSize(), mtimeStr, f.GetPath(), lineEnd)
|
||||
} else {
|
||||
_, _ = fmt.Fprintf(mfa.Stdout, "%s%s", f.GetPath(), lineEnd)
|
||||
}
|
||||
|
||||
+6
-18
@@ -21,14 +21,12 @@ const (
|
||||
cmdFreshen = "freshen"
|
||||
cmdExport = "export"
|
||||
cmdFetch = "fetch"
|
||||
cmdList = "list"
|
||||
cmdVersion = "version"
|
||||
|
||||
flagProgress = "progress"
|
||||
flagTimeout = "timeout"
|
||||
flagDest = "dest"
|
||||
flagRequireSignature = "require-signature"
|
||||
flagIncludePermissions = "include-permissions"
|
||||
|
||||
manifestArgsUsage = "[manifest file]"
|
||||
|
||||
@@ -169,16 +167,6 @@ func requireSignatureFlag() *cli.StringFlag {
|
||||
}
|
||||
}
|
||||
|
||||
// includePermissionsFlag returns the --include-permissions flag taken by the
|
||||
// generate and freshen subcommands.
|
||||
func includePermissionsFlag() *cli.BoolFlag {
|
||||
return &cli.BoolFlag{
|
||||
Name: flagIncludePermissions,
|
||||
Usage: "Record each file's permission bits in manifest " +
|
||||
"(recorded as 0000 by default)",
|
||||
}
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdGenerate,
|
||||
@@ -236,7 +224,6 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
Usage: "Include createdAt timestamp in manifest " +
|
||||
"(omitted by default for determinism)",
|
||||
},
|
||||
includePermissionsFlag(),
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -320,7 +307,6 @@ func (mfa *CLIApp) freshenCommand() *cli.Command {
|
||||
Usage: "Include createdAt timestamp in manifest " +
|
||||
"(omitted by default for determinism)",
|
||||
},
|
||||
includePermissionsFlag(),
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -354,7 +340,7 @@ func (mfa *CLIApp) versionCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) listCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdList,
|
||||
Name: "list",
|
||||
Aliases: []string{"ls"},
|
||||
Usage: "List files in manifest",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
@@ -364,7 +350,7 @@ func (mfa *CLIApp) listCommand() *cli.Command {
|
||||
&cli.BoolFlag{
|
||||
Name: "long",
|
||||
Aliases: []string{"l"},
|
||||
Usage: "Show mode, size and mtime",
|
||||
Usage: "Show size and mtime",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "print0",
|
||||
@@ -436,8 +422,10 @@ func (mfa *CLIApp) run(args []string) {
|
||||
Version: mfa.VersionString(),
|
||||
EnableShellCompletion: true,
|
||||
Writer: mfa.Stdout,
|
||||
// v3 writes its "Incorrect Usage" line to ErrWriter; v2 wrote it to
|
||||
// stdout, before the help, so it stays on stdout.
|
||||
// urfave/cli writes only its "Incorrect Usage" line to ErrWriter. It
|
||||
// goes to stdout with the help printed after it; run logs the error
|
||||
// itself to stderr. Tests rely on the logger being the only writer to
|
||||
// a run's stderr.
|
||||
ErrWriter: mfa.Stdout,
|
||||
Flags: commonFlags(),
|
||||
StopOnNthArg: stopOnFirstArg(),
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -138,14 +137,12 @@ func (b *Builder) SetSeed(seed string) {
|
||||
}
|
||||
|
||||
// AddFile reads file content from reader, computes hashes, and adds to manifest.
|
||||
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
||||
// Progress updates are sent to the progress channel (if non-nil) without blocking.
|
||||
// Returns the number of bytes read.
|
||||
func (b *Builder) AddFile(
|
||||
path RelFilePath,
|
||||
size FileSize,
|
||||
mtime ModTime,
|
||||
mode fs.FileMode,
|
||||
reader io.Reader,
|
||||
progress chan<- FileHashProgress,
|
||||
) (FileSize, error) {
|
||||
@@ -201,7 +198,6 @@ func (b *Builder) AddFile(
|
||||
{MultiHash: mh},
|
||||
},
|
||||
Mtime: mtime.Timestamp(),
|
||||
Mode: uint32(mode.Perm()),
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
@@ -233,14 +229,12 @@ func (b *Builder) FileCount() int {
|
||||
|
||||
// AddFileWithHash adds a file entry with a pre-computed hash.
|
||||
// This is useful when the hash is already known (e.g., from an existing manifest).
|
||||
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
||||
// Returns an error if path is invalid, size is negative, or hash is not a
|
||||
// multihash with a digest of at least 32 bytes, as long as SHA-256's.
|
||||
func (b *Builder) AddFileWithHash(
|
||||
path RelFilePath,
|
||||
size FileSize,
|
||||
mtime ModTime,
|
||||
mode fs.FileMode,
|
||||
hash Multihash,
|
||||
) error {
|
||||
err := ValidatePath(string(path))
|
||||
@@ -274,7 +268,6 @@ func (b *Builder) AddFileWithHash(
|
||||
{MultiHash: hash},
|
||||
},
|
||||
Mtime: mtime.Timestamp(),
|
||||
Mode: uint32(mode.Perm()),
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
|
||||
+18
-18
@@ -35,7 +35,7 @@ func TestBuilderAddFile(t *testing.T) {
|
||||
reader := bytes.NewReader(content)
|
||||
|
||||
bytesRead, err := b.AddFile(
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), 0, reader, nil,
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, FileSize(len(content)), bytesRead)
|
||||
@@ -49,7 +49,7 @@ func TestBuilderAddFileWithHash(t *testing.T) {
|
||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), 0, hash)
|
||||
err = b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, b.FileCount())
|
||||
}
|
||||
@@ -64,7 +64,7 @@ func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("", 100, ModTime(time.Now()), 0, sha256Hash)
|
||||
err := b.AddFileWithHash("", 100, ModTime(time.Now()), sha256Hash)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "path")
|
||||
})
|
||||
@@ -73,7 +73,7 @@ func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), 0, sha256Hash)
|
||||
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), sha256Hash)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "size")
|
||||
})
|
||||
@@ -82,7 +82,7 @@ func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), 0, sha256Hash)
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), sha256Hash)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, b.FileCount())
|
||||
})
|
||||
@@ -118,7 +118,7 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), 0, tt.hash)
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), tt.hash)
|
||||
require.ErrorIs(t, err, tt.want)
|
||||
assert.Equal(t, 0, b.FileCount())
|
||||
})
|
||||
@@ -133,7 +133,7 @@ func TestBuilderBuild(t *testing.T) {
|
||||
reader := bytes.NewReader(content)
|
||||
|
||||
_, err := b.AddFile(
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), 0, reader, nil,
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -196,7 +196,7 @@ func TestBuilderDeterministicOutput(t *testing.T) {
|
||||
for _, f := range files {
|
||||
r := bytes.NewReader([]byte(f.content))
|
||||
_, err := b.AddFile(
|
||||
RelFilePath(f.path), FileSize(len(f.content)), mtime, 0, r, nil,
|
||||
RelFilePath(f.path), FileSize(len(f.content)), mtime, r, nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -279,7 +279,7 @@ func TestBuilderAddFileSizeMismatch(t *testing.T) {
|
||||
reader := bytes.NewReader(content)
|
||||
|
||||
// Declare wrong size
|
||||
_, err := b.AddFile("test.txt", FileSize(100), ModTime(time.Now()), 0, reader, nil)
|
||||
_, err := b.AddFile("test.txt", FileSize(100), ModTime(time.Now()), reader, nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "size mismatch")
|
||||
}
|
||||
@@ -291,12 +291,12 @@ func TestBuilderAddFileInvalidPath(t *testing.T) {
|
||||
content := []byte("data")
|
||||
reader := bytes.NewReader(content)
|
||||
|
||||
_, err := b.AddFile("", FileSize(len(content)), ModTime(time.Now()), 0, reader, nil)
|
||||
_, err := b.AddFile("", FileSize(len(content)), ModTime(time.Now()), reader, nil)
|
||||
require.Error(t, err)
|
||||
|
||||
reader.Reset(content)
|
||||
_, err = b.AddFile(
|
||||
"/absolute", FileSize(len(content)), ModTime(time.Now()), 0, reader, nil,
|
||||
"/absolute", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
||||
)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -310,7 +310,7 @@ func TestBuilderAddFileWithProgress(t *testing.T) {
|
||||
progress := make(chan FileHashProgress, 100)
|
||||
|
||||
bytesRead, err := b.AddFile(
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), 0, reader, progress,
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, progress,
|
||||
)
|
||||
close(progress)
|
||||
require.NoError(t, err)
|
||||
@@ -345,7 +345,7 @@ func TestBuilderBuildRoundTrip(t *testing.T) {
|
||||
for _, f := range files {
|
||||
reader := bytes.NewReader(f.content)
|
||||
_, err := b.AddFile(
|
||||
RelFilePath(f.path), FileSize(len(f.content)), ModTime(now), 0, reader, nil,
|
||||
RelFilePath(f.path), FileSize(len(f.content)), ModTime(now), reader, nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -387,7 +387,7 @@ func TestBuilderBuildRoundTripLargeManifest(t *testing.T) {
|
||||
|
||||
for i := range 4000 {
|
||||
path := RelFilePath(fmt.Sprintf("dir/file-%05d.txt", i))
|
||||
require.NoError(t, b.AddFileWithHash(path, FileSize(i), ModTime{}, 0, hash))
|
||||
require.NoError(t, b.AddFileWithHash(path, FileSize(i), ModTime{}, hash))
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
@@ -452,7 +452,7 @@ func TestManifestString(t *testing.T) {
|
||||
content := []byte("test")
|
||||
reader := bytes.NewReader(content)
|
||||
_, err := b.AddFile(
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), 0, reader, nil,
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -484,7 +484,7 @@ func TestBuilderOmitsCreatedAtByDefault(t *testing.T) {
|
||||
b := NewBuilder()
|
||||
content := []byte("hello")
|
||||
_, err := b.AddFile(
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), 0,
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()),
|
||||
bytes.NewReader(content), nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
@@ -506,7 +506,7 @@ func TestBuilderIncludesCreatedAtWhenRequested(t *testing.T) {
|
||||
|
||||
content := []byte("hello")
|
||||
_, err := b.AddFile(
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), 0,
|
||||
"test.txt", FileSize(len(content)), ModTime(time.Now()),
|
||||
bytes.NewReader(content), nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
@@ -532,7 +532,7 @@ func TestBuilderDeterministicFileOrder(t *testing.T) {
|
||||
content := []byte("content of " + name)
|
||||
_, err := b.AddFile(
|
||||
RelFilePath(name), FileSize(len(content)),
|
||||
ModTime(time.Unix(1000, 0)), 0, bytes.NewReader(content), nil,
|
||||
ModTime(time.Unix(1000, 0)), bytes.NewReader(content), nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
+3
-17
@@ -36,7 +36,6 @@ const (
|
||||
StatusMissing // File not found on disk
|
||||
StatusSizeMismatch // File size differs from manifest
|
||||
StatusHashMismatch // File hash differs from manifest
|
||||
StatusModeMismatch // File permission bits differ from a recorded mode
|
||||
StatusExtra // File exists on disk but not in manifest
|
||||
StatusError // Error occurred during verification
|
||||
)
|
||||
@@ -51,8 +50,6 @@ func (s Status) String() string {
|
||||
return "SIZE_MISMATCH"
|
||||
case StatusHashMismatch:
|
||||
return "HASH_MISMATCH"
|
||||
case StatusModeMismatch:
|
||||
return "MODE_MISMATCH"
|
||||
case StatusExtra:
|
||||
return "EXTRA"
|
||||
case StatusError:
|
||||
@@ -411,23 +408,12 @@ func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
|
||||
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
|
||||
}
|
||||
|
||||
// Check against all hashes in manifest (at least one must match),
|
||||
// then against the recorded mode, where one is: 0 means none was.
|
||||
// Check against all hashes in manifest (at least one must match)
|
||||
for _, hash := range entry.GetHashes() {
|
||||
if !bytes.Equal(computed, hash.GetMultiHash()) {
|
||||
continue
|
||||
}
|
||||
|
||||
if entry.GetMode() != 0 && info.Mode().Perm() != os.FileMode(entry.GetMode()) {
|
||||
return Result{
|
||||
Path: relPath,
|
||||
Status: StatusModeMismatch,
|
||||
Message: "mode mismatch",
|
||||
}
|
||||
}
|
||||
|
||||
if bytes.Equal(computed, hash.GetMultiHash()) {
|
||||
return Result{Path: relPath, Status: StatusOK}
|
||||
}
|
||||
}
|
||||
|
||||
return Result{
|
||||
Path: relPath,
|
||||
|
||||
+2
-52
@@ -34,7 +34,6 @@ func TestStatusString(t *testing.T) {
|
||||
{StatusMissing, "MISSING"},
|
||||
{StatusSizeMismatch, "SIZE_MISMATCH"},
|
||||
{StatusHashMismatch, "HASH_MISMATCH"},
|
||||
{StatusModeMismatch, "MODE_MISMATCH"},
|
||||
{StatusExtra, "EXTRA"},
|
||||
{StatusError, "ERROR"},
|
||||
{Status(99), "UNKNOWN"},
|
||||
@@ -60,7 +59,7 @@ func createTestManifest(
|
||||
for path, content := range files {
|
||||
reader := bytes.NewReader(content)
|
||||
_, err := builder.AddFile(
|
||||
RelFilePath(path), FileSize(len(content)), ModTime(time.Now()), 0, reader, nil,
|
||||
RelFilePath(path), FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -280,8 +279,7 @@ func TestCheckMissingFile(t *testing.T) {
|
||||
missingCount++
|
||||
|
||||
assert.Equal(t, RelFilePath("missing.txt"), r.Path)
|
||||
case StatusSizeMismatch, StatusHashMismatch, StatusModeMismatch,
|
||||
StatusExtra, StatusError:
|
||||
case StatusSizeMismatch, StatusHashMismatch, StatusExtra, StatusError:
|
||||
// Not expected in this test; counted assertions below will fail.
|
||||
}
|
||||
}
|
||||
@@ -351,54 +349,6 @@ func TestCheckHashMismatch(t *testing.T) {
|
||||
assert.Equal(t, RelFilePath(testFileName), r.Path)
|
||||
}
|
||||
|
||||
// A recorded mode other than 0000 that differs from the file's permission
|
||||
// bits fails the check; a recorded 0000 is never checked.
|
||||
func TestCheckMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
recorded os.FileMode
|
||||
onDisk os.FileMode
|
||||
want Status
|
||||
}{
|
||||
{"recorded mode matches", 0o640, 0o640, StatusOK},
|
||||
{"recorded mode differs", 0o640, 0o600, StatusModeMismatch},
|
||||
{"0000 is not checked", 0, 0o600, StatusOK},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
content := []byte("content")
|
||||
|
||||
b := NewBuilder()
|
||||
_, err := b.AddFile(testFileName, FileSize(len(content)), ModTime{},
|
||||
tc.recorded, bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
var buf bytes.Buffer
|
||||
require.NoError(t, b.Build(context.Background(), &buf))
|
||||
require.NoError(t, afero.WriteFile(fs, testManifestPath, buf.Bytes(), 0o644))
|
||||
require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs,
|
||||
testDataDir+"/"+testFileName, content, tc.onDisk))
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 1)
|
||||
require.NoError(t, chk.Check(context.Background(), results, nil))
|
||||
|
||||
assert.Equal(t, tc.want, (<-results).Status)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckWithProgress(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+4
-4
@@ -29,8 +29,8 @@ const (
|
||||
// Bytes decoding sets aside for each file entry, hash, timestamp and
|
||||
// MIME type, however short its encoding. checkDecodedSize refuses an
|
||||
// inner message for which these add up to more than maxDecodedGrowth
|
||||
// times its size. The mode is held in the file entry itself.
|
||||
decodedFileEntrySize = 176
|
||||
// times its size.
|
||||
decodedFileEntrySize = 160
|
||||
decodedHashSize = 112
|
||||
decodedTimestampSize = 64
|
||||
decodedMIMETypeSize = 16
|
||||
@@ -38,7 +38,7 @@ const (
|
||||
// Each file entry mfer writes holds a path of at least one byte, a
|
||||
// multihash at least as long as SHA-256's 34 bytes (AddFileWithHash
|
||||
// refuses shorter ones) and a modification time: at least 47 bytes,
|
||||
// counted at 352. So its manifests add up to at most about 7.49 times
|
||||
// their size, and this limit is about 7% above that.
|
||||
// counted at 336. So its manifests add up to at most about 7.15 times
|
||||
// their size, and this limit is about 12% above that.
|
||||
maxDecodedGrowth = 8
|
||||
)
|
||||
|
||||
@@ -119,11 +119,11 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
||||
}
|
||||
|
||||
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
||||
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
|
||||
// and take 16 bytes plus the path to encode. A 37-character path makes that
|
||||
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
|
||||
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
|
||||
// it 55 bytes, about 7.9 times: loaded.
|
||||
// and change times are counted at 416 bytes each (160 + 112 + 16 + 64 + 64)
|
||||
// and take 16 bytes plus the path to encode. A 35-character path makes that
|
||||
// 51 bytes, about 8.2 times: refused, and leaving any one of the five
|
||||
// uncounted, even the MIME type, brings it under 8. A 37-character path makes
|
||||
// it 53 bytes, about 7.8 times: loaded.
|
||||
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -131,8 +131,8 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||
pathLen int
|
||||
refused bool
|
||||
}{
|
||||
{37, true},
|
||||
{39, false},
|
||||
{35, true},
|
||||
{37, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -215,7 +215,7 @@ func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
||||
const files = 10000
|
||||
for i := range files {
|
||||
name := RelFilePath(strconv.FormatInt(int64(i), 36))
|
||||
require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), 0, hash))
|
||||
require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), hash))
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
@@ -233,7 +233,7 @@ func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
b := NewBuilder()
|
||||
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, 0, hash))
|
||||
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, hash))
|
||||
|
||||
var buf bytes.Buffer
|
||||
require.NoError(t, b.Build(context.Background(), &buf))
|
||||
|
||||
+4
-4
@@ -187,7 +187,7 @@ func TestBuilderWithSigning(t *testing.T) {
|
||||
// Add a test file
|
||||
content := []byte("test file content")
|
||||
reader := bytes.NewReader(content)
|
||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, 0, reader, nil)
|
||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, reader, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Build the manifest
|
||||
@@ -314,7 +314,7 @@ func TestManifestSignatureVerification(t *testing.T) {
|
||||
// Add a test file
|
||||
content := []byte("test file content for verification")
|
||||
reader := bytes.NewReader(content)
|
||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, 0, reader, nil)
|
||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, reader, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Build the manifest
|
||||
@@ -344,7 +344,7 @@ func TestManifestTamperedSignatureFails(t *testing.T) {
|
||||
|
||||
content := []byte("test file content")
|
||||
reader := bytes.NewReader(content)
|
||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, 0, reader, nil)
|
||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, reader, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
var buf bytes.Buffer
|
||||
@@ -377,7 +377,7 @@ func TestBuilderWithoutSigning(t *testing.T) {
|
||||
// Add a test file
|
||||
content := []byte("test file content")
|
||||
reader := bytes.NewReader(content)
|
||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, 0, reader, nil)
|
||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, reader, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Build the manifest
|
||||
|
||||
+2
-12
@@ -340,8 +340,6 @@ type MFFilePath struct {
|
||||
MimeType *string `protobuf:"bytes,301,opt,name=mimeType,proto3,oneof" json:"mimeType,omitempty"`
|
||||
Mtime *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"`
|
||||
Ctime *Timestamp `protobuf:"bytes,303,opt,name=ctime,proto3,oneof" json:"ctime,omitempty"`
|
||||
// permission bits, at most 0777; 0 when not recorded
|
||||
Mode uint32 `protobuf:"varint,304,opt,name=mode,proto3" json:"mode,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -418,13 +416,6 @@ func (x *MFFilePath) GetCtime() *Timestamp {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *MFFilePath) GetMode() uint32 {
|
||||
if x != nil {
|
||||
return x.Mode
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type MFFileChecksum struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// 1.0 golang implementation must write a multihash here
|
||||
@@ -570,7 +561,7 @@ const file_mf_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"_signatureB\t\n" +
|
||||
"\a_signerB\x10\n" +
|
||||
"\x0e_signingPubKey\"\x85\x02\n" +
|
||||
"\x0e_signingPubKey\"\xf0\x01\n" +
|
||||
"\n" +
|
||||
"MFFilePath\x12\x12\n" +
|
||||
"\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" +
|
||||
@@ -580,8 +571,7 @@ const file_mf_proto_rawDesc = "" +
|
||||
"\x05mtime\x18\xae\x02 \x01(\v2\n" +
|
||||
".TimestampH\x01R\x05mtime\x88\x01\x01\x12&\n" +
|
||||
"\x05ctime\x18\xaf\x02 \x01(\v2\n" +
|
||||
".TimestampH\x02R\x05ctime\x88\x01\x01\x12\x13\n" +
|
||||
"\x04mode\x18\xb0\x02 \x01(\rR\x04modeB\v\n" +
|
||||
".TimestampH\x02R\x05ctime\x88\x01\x01B\v\n" +
|
||||
"\t_mimeTypeB\b\n" +
|
||||
"\x06_mtimeB\b\n" +
|
||||
"\x06_ctime\".\n" +
|
||||
|
||||
@@ -59,8 +59,6 @@ message MFFilePath {
|
||||
optional string mimeType = 301;
|
||||
optional Timestamp mtime = 302;
|
||||
optional Timestamp ctime = 303;
|
||||
// permission bits, at most 0777; 0 when not recorded
|
||||
uint32 mode = 304;
|
||||
}
|
||||
|
||||
message MFFileChecksum {
|
||||
|
||||
@@ -1 +1 @@
|
||||
3d4dcb0b2f4640dd3ae6bb48b833e9f26ae9771e2d3e88bd646e7f1724dda654 mf.proto
|
||||
fa6fceaba5553c8667c631994535fe5c307c8adb19f3ad289babf79a0f438650 mf.proto
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/reflect/protoreflect"
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
// mf.pb.go is generated from mf.proto and committed. `make generate`
|
||||
@@ -29,27 +27,3 @@ func TestGeneratedCodeMatchesProto(t *testing.T) {
|
||||
"mfer/mf.proto has changed since mfer/mf.pb.go was generated "+
|
||||
"from it: run `make generate` and commit the result")
|
||||
}
|
||||
|
||||
// A file entry has exactly the fields docs/FORMAT.md lists for MFFilePath.
|
||||
func TestFileEntryFieldsMatchSpec(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
want := map[string]protoreflect.FieldNumber{
|
||||
"path": 1,
|
||||
"size": 2,
|
||||
"hashes": 3,
|
||||
"mimeType": 301,
|
||||
"mtime": 302,
|
||||
"ctime": 303,
|
||||
"mode": 304,
|
||||
}
|
||||
|
||||
got := map[string]protoreflect.FieldNumber{}
|
||||
|
||||
fields := (&mfer.MFFilePath{}).ProtoReflect().Descriptor().Fields()
|
||||
for i := range fields.Len() {
|
||||
got[string(fields.Get(i).Name())] = fields.Get(i).Number()
|
||||
}
|
||||
|
||||
require.Equal(t, want, got)
|
||||
}
|
||||
|
||||
@@ -52,9 +52,6 @@ type ScannerOptions struct {
|
||||
// IncludeTimestamps includes a createdAt timestamp in the manifest
|
||||
// (default: omit for determinism).
|
||||
IncludeTimestamps bool
|
||||
// IncludePermissions records each file's permission bits, 0777 at
|
||||
// most, in the manifest (default: record 0000).
|
||||
IncludePermissions bool
|
||||
// Fs is the filesystem to use, defaults to OsFs if nil.
|
||||
Fs afero.Fs
|
||||
// SigningOptions holds GPG signing options (nil = no signing).
|
||||
@@ -72,7 +69,6 @@ type FileEntry struct {
|
||||
Size FileSize // File size in bytes
|
||||
Mtime ModTime // Last modification time
|
||||
Ctime time.Time // Creation time (platform-dependent)
|
||||
Mode fs.FileMode // Permission bits (Perm() of the file's mode)
|
||||
}
|
||||
|
||||
// Scanner accumulates files and generates manifests from them.
|
||||
@@ -357,18 +353,11 @@ func (s *Scanner) scanFile(
|
||||
}(scannedBytes, scannedFiles)
|
||||
}
|
||||
|
||||
// A mode of 0 records 0000, which means none was recorded.
|
||||
var mode fs.FileMode
|
||||
if s.options.IncludePermissions {
|
||||
mode = entry.Mode
|
||||
}
|
||||
|
||||
// Add to manifest with progress channel
|
||||
bytesRead, err := builder.AddFile(
|
||||
entry.Path,
|
||||
entry.Size,
|
||||
entry.Mtime,
|
||||
mode,
|
||||
f,
|
||||
fileProgress,
|
||||
)
|
||||
@@ -472,7 +461,6 @@ func (s *Scanner) enumerateFileWithInfo(
|
||||
AbsPath: AbsFilePath(absPath),
|
||||
Size: FileSize(info.Size()),
|
||||
Mtime: ModTime(info.ModTime()),
|
||||
Mode: info.Mode().Perm(),
|
||||
// Note: Ctime not available from fs.FileInfo on all platforms
|
||||
// Will need platform-specific code to extract it
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ package mfer
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -351,46 +350,6 @@ func TestScannerFileEntryFields(t *testing.T) {
|
||||
assert.WithinDuration(t, now, time.Time(entry.Mtime), 2*time.Second)
|
||||
}
|
||||
|
||||
// A manifest records every mode as 0000 unless the creator asks for
|
||||
// permissions; then it records each file's permission bits and never the
|
||||
// setuid, setgid or sticky bits.
|
||||
func TestScannerRecordsModeOnlyWhenAsked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, afero.WriteFile(fs, "/"+testFile1, []byte("a"), 0o640))
|
||||
require.NoError(t, afero.WriteFile(fs, "/run.sh", []byte("b"), 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, "/su", []byte("c"), 0o755))
|
||||
require.NoError(t, fs.Chmod("/su", 0o755|os.ModeSetuid|os.ModeSetgid|os.ModeSticky))
|
||||
|
||||
for _, tc := range []struct {
|
||||
includePermissions bool
|
||||
want map[string]uint32
|
||||
}{
|
||||
{false, map[string]uint32{testFile1: 0, "run.sh": 0, "su": 0}},
|
||||
{true, map[string]uint32{testFile1: 0o640, "run.sh": 0o755, "su": 0o755}},
|
||||
} {
|
||||
s := NewScannerWithOptions(&ScannerOptions{
|
||||
Fs: fs,
|
||||
IncludePermissions: tc.includePermissions,
|
||||
})
|
||||
require.NoError(t, s.EnumerateFS(fs, "/", nil))
|
||||
|
||||
var buf bytes.Buffer
|
||||
require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
|
||||
|
||||
m, err := NewManifestFromReader(&buf)
|
||||
require.NoError(t, err)
|
||||
|
||||
got := map[string]uint32{}
|
||||
for _, f := range m.Files() {
|
||||
got[f.GetPath()] = f.GetMode()
|
||||
}
|
||||
|
||||
assert.Equal(t, tc.want, got, "IncludePermissions: %v", tc.includePermissions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerLargeFileEnumeration(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+4
-60
@@ -22,14 +22,6 @@ YARN_VERSION="1.22.22"
|
||||
# protoc v33.4, 2026-10-04, for script/generate. The sha256 of each
|
||||
# platform's release archive is in ensure_protoc.
|
||||
PROTOC_VERSION="33.4"
|
||||
# gofumpt v0.12.0 for script/gofumpt and protoc-gen-go v1.36.11 for
|
||||
# script/generate, 2026-10-04: each is installed into bin/ with
|
||||
# `go install`, pinned to the commit its release tag names. Those two
|
||||
# scripts refuse any other version, so a new pin is changed there too.
|
||||
GOFUMPT_VERSION="v0.12.0"
|
||||
GOFUMPT_COMMIT="3e07e7e70ac93761d8e79ca0083a19e3d59f753d"
|
||||
PROTOC_GEN_GO_VERSION="v1.36.11"
|
||||
PROTOC_GEN_GO_COMMIT="96a179180f0ad6bba9b1e7b6e38d0affb0168e9a"
|
||||
|
||||
PKGMGR=""
|
||||
SUDO=""
|
||||
@@ -138,13 +130,11 @@ install_js_deps() {
|
||||
|
||||
# Unpack protoc's release archive for this platform into bin/protoc, after
|
||||
# checking the archive's sha256, unless bin/protoc already holds the pinned
|
||||
# version. script/generate runs bin/protoc/bin/protoc, so that is the
|
||||
# binary checked again after unpacking.
|
||||
# version.
|
||||
ensure_protoc() {
|
||||
dir="$ROOT/bin/protoc"
|
||||
if [ "$("$dir/bin/protoc" --version 2>/dev/null)" = \
|
||||
"libprotoc $PROTOC_VERSION" ]; then
|
||||
echo "protoc $PROTOC_VERSION"
|
||||
return 0
|
||||
fi
|
||||
case "$(uname -s) $(uname -m)" in
|
||||
@@ -178,53 +168,6 @@ ensure_protoc() {
|
||||
rm -rf "$dir"
|
||||
unzip -q "$tmp/protoc.zip" -d "$dir"
|
||||
rm -rf "$tmp"
|
||||
actual="$("$dir/bin/protoc" --version 2>/dev/null || true)"
|
||||
if [ "$actual" != "libprotoc $PROTOC_VERSION" ]; then
|
||||
echo "bootstrap: $dir/bin/protoc reports '$actual'," \
|
||||
"not libprotoc $PROTOC_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "protoc $PROTOC_VERSION"
|
||||
}
|
||||
|
||||
# Install gofumpt into bin/ unless bin/gofumpt already reports the pinned
|
||||
# version. script/gofumpt runs bin/gofumpt, so that is the binary checked
|
||||
# again after installing. Its --version prints the version, then the Go
|
||||
# version it was built with. The old file is removed first because
|
||||
# `go install` refuses to replace a file that is not a Go binary.
|
||||
ensure_gofumpt() {
|
||||
tool="$ROOT/bin/gofumpt"
|
||||
if [ "$("$tool" --version 2>/dev/null | cut -d' ' -f1)" != \
|
||||
"$GOFUMPT_VERSION" ]; then
|
||||
rm -f "$tool"
|
||||
GOBIN="$ROOT/bin" go install "mvdan.cc/gofumpt@$GOFUMPT_COMMIT"
|
||||
fi
|
||||
actual="$("$tool" --version 2>/dev/null | cut -d' ' -f1)"
|
||||
if [ "$actual" != "$GOFUMPT_VERSION" ]; then
|
||||
echo "bootstrap: $tool reports '$actual', not $GOFUMPT_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "gofumpt $GOFUMPT_VERSION"
|
||||
}
|
||||
|
||||
# Install protoc-gen-go into bin/ unless bin/protoc-gen-go already reports
|
||||
# the pinned version, as ensure_gofumpt does. script/generate runs
|
||||
# bin/protoc-gen-go, so that is the binary checked again after installing.
|
||||
ensure_protoc_gen_go() {
|
||||
tool="$ROOT/bin/protoc-gen-go"
|
||||
if [ "$("$tool" --version 2>/dev/null)" != \
|
||||
"protoc-gen-go $PROTOC_GEN_GO_VERSION" ]; then
|
||||
rm -f "$tool"
|
||||
GOBIN="$ROOT/bin" go install \
|
||||
"google.golang.org/protobuf/cmd/protoc-gen-go@$PROTOC_GEN_GO_COMMIT"
|
||||
fi
|
||||
actual="$("$tool" --version 2>/dev/null || true)"
|
||||
if [ "$actual" != "protoc-gen-go $PROTOC_GEN_GO_VERSION" ]; then
|
||||
echo "bootstrap: $tool reports '$actual'," \
|
||||
"not protoc-gen-go $PROTOC_GEN_GO_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "protoc-gen-go $PROTOC_GEN_GO_VERSION"
|
||||
}
|
||||
|
||||
main() {
|
||||
@@ -249,9 +192,10 @@ main() {
|
||||
if missing go; then pkg_install go golang go go; fi
|
||||
# No golangci-lint: script/lint runs it in Docker only.
|
||||
go mod download
|
||||
ensure_gofumpt
|
||||
# gofumpt and protoc-gen-go: bin/tools/go.mod pins them, and
|
||||
# script/gofumpt and script/generate build them from there.
|
||||
(cd "$ROOT/bin/tools" && go mod download)
|
||||
ensure_protoc
|
||||
ensure_protoc_gen_go
|
||||
|
||||
# ---- Python repos ----
|
||||
# if missing python3; then pkg_install python3 python3 python3 python3; fi
|
||||
|
||||
+2
-3
@@ -1,8 +1,7 @@
|
||||
#!/bin/sh
|
||||
# script/check: run all checks (test, lint, fmt-check). Our own
|
||||
# extension to scripts-to-rule-them-all. test and lint are Docker
|
||||
# phases; fmt-check is native, because a formatter writes the working
|
||||
# tree. Must not modify any files.
|
||||
# extension to scripts-to-rule-them-all. Must not modify any files.
|
||||
# Generic: usually needs no adaptation.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
|
||||
+4
-8
@@ -1,10 +1,8 @@
|
||||
#!/bin/sh
|
||||
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
||||
# checks out and runs this and nothing else, and script/fmt-check runs
|
||||
# the formatter on the host, which a pristine checkout cannot do.
|
||||
# --no-cache for the same reason as script/docker: the gate phases the
|
||||
# final stage depends on are RUN steps, and a cached one is a check that
|
||||
# did not run.
|
||||
# script/cibuild: run the CI build; the Gitea workflow runs this on push.
|
||||
# It builds the image with the same command as script/docker. --no-cache
|
||||
# because the checks the final stage depends on are RUN steps, and a
|
||||
# cached one is a check that did not run.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
@@ -12,8 +10,6 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
"$SCRIPT_DIR/bootstrap"
|
||||
"$SCRIPT_DIR/check"
|
||||
# Own line: a failing command substitution inside an argument does
|
||||
# not trip `set -e`, so the inline form degrades silently to an
|
||||
# empty constant. The VERSION build argument takes precedence over
|
||||
|
||||
+10
-15
@@ -5,18 +5,16 @@
|
||||
# protoc. A test fails while mf.proto no longer matches the recorded hash.
|
||||
#
|
||||
# Runs the protoc that script/bootstrap unpacks into bin/protoc, and the
|
||||
# protoc-gen-go it installs into bin/. Another version of either writes a
|
||||
# different mf.pb.go.
|
||||
# protoc-gen-go that bin/tools/go.mod pins. Another version of either
|
||||
# writes a different mf.pb.go.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# The versions script/bootstrap installs. protoc 33.4 names itself v6.33.4
|
||||
# in the mf.pb.go header.
|
||||
# The protoc version script/bootstrap installs. protoc 33.4 names itself
|
||||
# v6.33.4 in the mf.pb.go header.
|
||||
PROTOC_VERSION="33.4"
|
||||
PROTOC="$ROOT/bin/protoc/bin/protoc"
|
||||
PROTOC_GEN_GO_VERSION="v1.36.11"
|
||||
PROTOC_GEN_GO="$ROOT/bin/protoc-gen-go"
|
||||
|
||||
# sha256 <file>: print "<hash> <file>", with sha256sum, or with shasum
|
||||
# where there is no sha256sum.
|
||||
@@ -32,26 +30,23 @@ sha256() {
|
||||
}
|
||||
|
||||
main() {
|
||||
# A bin/protoc or bin/protoc-gen-go left from before its pin moved
|
||||
# fails here, until script/bootstrap replaces it.
|
||||
# A bin/protoc left from before the pin moved fails here, until
|
||||
# script/bootstrap replaces it.
|
||||
actual="$("$PROTOC" --version 2>/dev/null || true)"
|
||||
if [ "$actual" != "libprotoc $PROTOC_VERSION" ]; then
|
||||
echo "generate: needs protoc $PROTOC_VERSION in bin/protoc," \
|
||||
"found: ${actual:-none}; run script/bootstrap" >&2
|
||||
exit 1
|
||||
fi
|
||||
actual="$("$PROTOC_GEN_GO" --version 2>/dev/null || true)"
|
||||
if [ "$actual" != "protoc-gen-go $PROTOC_GEN_GO_VERSION" ]; then
|
||||
echo "generate: needs protoc-gen-go $PROTOC_GEN_GO_VERSION in" \
|
||||
"bin/protoc-gen-go, found: ${actual:-none}; run script/bootstrap" >&2
|
||||
exit 1
|
||||
fi
|
||||
# `go tool -n` builds protoc-gen-go from bin/tools and prints where the
|
||||
# binary is, without running it.
|
||||
plugin="$(cd "$ROOT/bin/tools" && go tool -n protoc-gen-go)"
|
||||
|
||||
cd "$ROOT/mfer"
|
||||
# Hashed before regenerating, so a missing hash tool stops the script
|
||||
# before it changes anything. Regenerating leaves mf.proto as it is.
|
||||
proto_hash="$(sha256 mf.proto)"
|
||||
"$PROTOC" --plugin=protoc-gen-go="$PROTOC_GEN_GO" \
|
||||
"$PROTOC" --plugin=protoc-gen-go="$plugin" \
|
||||
--go_out=paths=source_relative:. ./mf.proto
|
||||
echo "$proto_hash" >mf.proto.sha256
|
||||
}
|
||||
|
||||
+10
-16
@@ -3,16 +3,17 @@
|
||||
#
|
||||
# Takes exactly one mode argument, --write or --check, and runs the same
|
||||
# gofumpt version over the same files in both modes. script/fmt and
|
||||
# script/fmt-check both go through here, so what gets formatted and what
|
||||
# gets verified cannot drift apart.
|
||||
# script/fmt-check both go through here, and so does the Docker lint
|
||||
# stage, so what gets formatted and what gets verified cannot drift
|
||||
# apart.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# The gofumpt script/bootstrap installs into bin/. Must match the pin
|
||||
# there.
|
||||
GOFUMPT_VERSION="v0.12.0"
|
||||
GOFUMPT="$ROOT/bin/gofumpt"
|
||||
# The gofumpt version is the one bin/tools/go.mod pins. `go tool` run in
|
||||
# bin/tools builds it from source checked against the hashes in
|
||||
# bin/tools/go.sum, so neither a developer machine nor the lint image needs
|
||||
# it installed.
|
||||
|
||||
usage() {
|
||||
echo "usage: script/gofumpt --write|--check" >&2
|
||||
@@ -21,22 +22,15 @@ usage() {
|
||||
|
||||
main() {
|
||||
[ "$#" -eq 1 ] || usage
|
||||
# A bin/gofumpt left from before the pin moved formats differently, so
|
||||
# it fails here until script/bootstrap replaces it.
|
||||
actual="$("$GOFUMPT" --version 2>/dev/null | cut -d' ' -f1)"
|
||||
if [ "$actual" != "$GOFUMPT_VERSION" ]; then
|
||||
echo "gofumpt: needs $GOFUMPT_VERSION in bin/gofumpt," \
|
||||
"found: ${actual:-none}; run script/bootstrap" >&2
|
||||
exit 1
|
||||
fi
|
||||
cd "$ROOT/bin/tools"
|
||||
# Every Go file in the repo, from $ROOT down. gofumpt holds generated
|
||||
# files, such as mfer/mf.pb.go, to gofmt's rules only.
|
||||
case "$1" in
|
||||
--write) "$GOFUMPT" -l -w "$ROOT" ;;
|
||||
--write) go tool gofumpt -l -w "$ROOT" ;;
|
||||
--check)
|
||||
# Own line: a failing command inside `[ -n "$(...)" ]` does
|
||||
# not trip `set -e`, so a gofumpt that never ran would pass.
|
||||
unformatted="$("$GOFUMPT" -l "$ROOT")"
|
||||
unformatted="$(go tool gofumpt -l "$ROOT")"
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "gofumpt: files need formatting (run make fmt):" >&2
|
||||
echo "$unformatted" >&2
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/bin/sh
|
||||
# script/install-precommit: install the git pre-commit hook that runs
|
||||
# script/precommit. Our own extension to scripts-to-rule-them-all.
|
||||
# Generic: needs no adaptation.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
+9
-12
@@ -1,13 +1,8 @@
|
||||
#!/bin/sh
|
||||
# script/lint: run the linter. Linting is a phase of the Dockerfile and
|
||||
# this builds that phase alone; the linter is never installed or run on
|
||||
# a developer host, where a shared result cache and a host-global lock
|
||||
# make its answer untrustworthy.
|
||||
#
|
||||
# The phase is not the last stage in the file, so it is built only when
|
||||
# --target names it. --no-cache because a cached lint layer is a lint
|
||||
# that did not run. The tag makes each build replace the previous image
|
||||
# instead of leaving a dangling one behind.
|
||||
# script/lint: run golangci-lint, in Docker only. Builds the lint stage of
|
||||
# the Dockerfile, whose build runs the linter, so a successful build is a
|
||||
# clean lint. --no-cache because a cached build runs no linter. The image
|
||||
# is removed afterwards, whatever the outcome.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
@@ -15,9 +10,11 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
docker build --no-cache \
|
||||
--target lint \
|
||||
-t "$("$SCRIPT_DIR/projectname")-lint" .
|
||||
# Tagged per run, so concurrent runs never remove each other's image.
|
||||
image="$("$SCRIPT_DIR/projectname")-lint:$$"
|
||||
# A failed build leaves no image, so there is nothing to remove then.
|
||||
trap 'docker image rm "$image" >/dev/null 2>&1 || true' EXIT INT TERM
|
||||
docker build --no-cache --target lint -t "$image" .
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+2
-1
@@ -56,7 +56,8 @@ main() {
|
||||
|
||||
# Markdown and JSON, repo-wide rather than root-only, so files in
|
||||
# subdirectories (docs/, once it exists) are covered too. Exclusions
|
||||
# live in .prettierignore.
|
||||
# live in .prettierignore; REPO_POLICIES.md is excluded there because
|
||||
# it is a verbatim copy of an upstream document.
|
||||
#
|
||||
# --no-error-on-unmatched-pattern is deliberately NOT used: both
|
||||
# patterns always match at least one tracked file (README.md,
|
||||
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
#!/bin/sh
|
||||
# script/setup: set up the repo for development after a fresh clone:
|
||||
# installs dependencies and the git pre-commit hook.
|
||||
# installs dependencies (script/bootstrap) and the git pre-commit hook.
|
||||
# Add any repo-specific initialization (db init, .env template) here.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
|
||||
+8
-10
@@ -1,19 +1,17 @@
|
||||
#!/bin/sh
|
||||
# script/test: run the test suite. Testing is a phase of the Dockerfile
|
||||
# and this builds that phase alone, on the same terms as script/lint:
|
||||
# --target because a phase that is not the last stage is built only when
|
||||
# named, --no-cache because a cached test layer is a test that did not
|
||||
# run, and a tag so each build replaces the previous image.
|
||||
# script/test: run the test suite.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
docker build --no-cache \
|
||||
--target test \
|
||||
-t "$("$SCRIPT_DIR/projectname")-test" .
|
||||
go test -timeout 30s -race -cover ./... ||
|
||||
{
|
||||
echo "--- Rerunning with -v for details ---"
|
||||
go test -timeout 30s -race -v ./...
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user