Compare commits
2
Commits
ef56eeeae0
...
239cccd45a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
239cccd45a | ||
|
|
9bb0ab3a03 |
@@ -61,3 +61,6 @@
|
|||||||
|
|
||||||
# This repo's own host-built binary (make build).
|
# This repo's own host-built binary (make build).
|
||||||
/bin/mfer
|
/bin/mfer
|
||||||
|
|
||||||
|
# The protoc script/bootstrap unpacks for script/generate.
|
||||||
|
/bin/protoc
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
/bin/mfer
|
/bin/mfer
|
||||||
|
/bin/protoc/
|
||||||
/tmp
|
/tmp
|
||||||
/node_modules/
|
/node_modules/
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -10,7 +10,7 @@ COPY . .
|
|||||||
|
|
||||||
# Go half of fmt-check only: this image has no node, so no prettier. The
|
# Go half of fmt-check only: this image has no node, so no prettier. The
|
||||||
# markdown half runs in the mdfmt stage below. The image has no gofumpt
|
# markdown half runs in the mdfmt stage below. The image has no gofumpt
|
||||||
# either; script/gofumpt builds the version it pins with `go run`.
|
# either; script/gofumpt builds the version bin/tools/go.mod pins.
|
||||||
RUN script/gofumpt --check
|
RUN script/gofumpt --check
|
||||||
# The linter directly, not `make lint`: script/lint builds this stage, and
|
# The linter directly, not `make lint`: script/lint builds this stage, and
|
||||||
# there is no docker inside this build.
|
# there is no docker inside this build.
|
||||||
@@ -18,7 +18,7 @@ RUN golangci-lint run --config .golangci.yml ./...
|
|||||||
|
|
||||||
# Markdown/JSON format stage — prettier needs node, which the Go images
|
# Markdown/JSON format stage — prettier needs node, which the Go images
|
||||||
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
||||||
# 22.17.0 and yarn 1.22.22, the versions script/bootstrap pins.
|
# 22.17.0 and yarn 1.22.22, the versions .nvmrc and script/bootstrap pin.
|
||||||
FROM node@sha256:b04ce4ae4e95b522112c2e5c52f781471a5cbc3b594527bcddedee9bc48c03a0 AS mdfmt
|
FROM node@sha256:b04ce4ae4e95b522112c2e5c52f781471a5cbc3b594527bcddedee9bc48c03a0 AS mdfmt
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
@@ -31,7 +31,7 @@ COPY . .
|
|||||||
RUN script/prettier --check
|
RUN script/prettier --check
|
||||||
|
|
||||||
# Build stage — tests and compilation
|
# Build stage — tests and compilation
|
||||||
# golang:1.23 (2026-03-14)
|
# golang:1.23.12, 2026-03-14
|
||||||
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
|
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
|
||||||
|
|
||||||
# Force BuildKit to run the lint and mdfmt stages by creating stage dependencies
|
# Force BuildKit to run the lint and mdfmt stages by creating stage dependencies
|
||||||
|
|||||||
@@ -67,9 +67,13 @@ standard: normalized scripts in `script/` are the entrypoints for the
|
|||||||
development workflow, and the Makefile targets are thin shims that call them. We
|
development workflow, and the Makefile targets are thin shims that call them. We
|
||||||
provide:
|
provide:
|
||||||
|
|
||||||
- `script/bootstrap` — install all dependencies (Go, Go module download, and
|
- `script/bootstrap` — install all dependencies, idempotently: Go and the
|
||||||
node/yarn plus the prettier version pinned in `package.json`/`yarn.lock`),
|
modules of both `go.mod` and `bin/tools/go.mod`; node (the version `.nvmrc`
|
||||||
idempotently; golangci-lint is not installed, it runs only in Docker
|
names, through nvm when there is no node on `PATH`) and yarn, plus the
|
||||||
|
prettier version pinned in `package.json`/`yarn.lock`; and `protoc` 33.4,
|
||||||
|
unpacked into `bin/protoc` from its release archive once the archive matches
|
||||||
|
the sha256 the script holds for this platform. golangci-lint is not installed,
|
||||||
|
it runs only in Docker
|
||||||
- `script/setup` — make a fresh clone ready for development: runs
|
- `script/setup` — make a fresh clone ready for development: runs
|
||||||
`script/bootstrap`, then `script/install-precommit`
|
`script/bootstrap`, then `script/install-precommit`
|
||||||
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
||||||
@@ -82,14 +86,10 @@ provide:
|
|||||||
- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from
|
- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from
|
||||||
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in
|
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in
|
||||||
`mfer/mf.proto.sha256`; the only thing that regenerates the committed
|
`mfer/mf.proto.sha256`; the only thing that regenerates the committed
|
||||||
`mfer/mf.pb.go`. It needs the exact versions that wrote the committed file,
|
`mfer/mf.pb.go`. It runs the `protoc` that `script/bootstrap` unpacks into
|
||||||
and refuses to run with any other: `protoc` 33.4 (unpack
|
`bin/protoc`, refusing any version but 33.4, and the `protoc-gen-go` that
|
||||||
`protoc-33.4-<platform>.zip` from
|
`bin/tools/go.mod` pins, which `go tool` builds from source checked against
|
||||||
[its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4)
|
the hashes in `bin/tools/go.sum`
|
||||||
and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11
|
|
||||||
(`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which
|
|
||||||
installs it in `$(go env GOPATH)/bin`; `script/generate` adds that directory
|
|
||||||
to `PATH`)
|
|
||||||
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
|
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
|
||||||
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
|
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
|
||||||
as ordinary tests
|
as ordinary tests
|
||||||
@@ -99,14 +99,17 @@ provide:
|
|||||||
- `script/fmt` — format all code and docs (writes): `script/gofumpt --write` and
|
- `script/fmt` — format all code and docs (writes): `script/gofumpt --write` and
|
||||||
`script/prettier --write`
|
`script/prettier --write`
|
||||||
- `script/gofumpt` — run `gofumpt` over every Go file in the repository in the
|
- `script/gofumpt` — run `gofumpt` over every Go file in the repository in the
|
||||||
given mode, `--write` or `--check`, at the one version it pins (built on
|
given mode, `--write` or `--check`, at the version `bin/tools/go.mod` pins
|
||||||
demand by `go run`, so nothing installs it); `script/fmt`, `script/fmt-check`
|
(built on demand by `go tool` from source checked against the hashes in
|
||||||
|
`bin/tools/go.sum`, so nothing installs it); `script/fmt`, `script/fmt-check`
|
||||||
and the Docker lint stage all go through it, so they cannot disagree about Go
|
and the Docker lint stage all go through it, so they cannot disagree about Go
|
||||||
formatting
|
formatting
|
||||||
- `script/prettier` — run prettier over the repository's canonical file set
|
- `script/prettier` — run prettier over the repository's canonical file set
|
||||||
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
||||||
`--check`; the single definition of that file set, so `script/fmt` and
|
`--check`, with the prettier version `yarn.lock` pins, run by the node on
|
||||||
`script/fmt-check` cannot disagree about it
|
`PATH` or else the one `script/bootstrap` installed through nvm; the single
|
||||||
|
definition of that file set, so `script/fmt` and `script/fmt-check` cannot
|
||||||
|
disagree about it
|
||||||
- `script/fmt-check` — check formatting without writing:
|
- `script/fmt-check` — check formatting without writing:
|
||||||
`script/gofumpt --check` plus `script/prettier --check`
|
`script/gofumpt --check` plus `script/prettier --check`
|
||||||
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
||||||
@@ -268,9 +271,12 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
|||||||
cryptographic integrity of downloaded files. A file already there with the
|
cryptographic integrity of downloaded files. A file already there with the
|
||||||
size and hash the manifest lists is skipped. Once every file is in place,
|
size and hash the manifest lists is skipped. Once every file is in place,
|
||||||
the manifest is saved there as `index.mf`, so `mfer check` can verify the
|
the manifest is saved there as `index.mf`, so `mfer check` can verify the
|
||||||
tree later. A manifest that lists `index.mf` (in any letter case) or
|
tree later. Each file is downloaded to a temp file beside it, such as
|
||||||
`.index.mf.tmp` at the top of the tree is refused before any file is
|
`.a.txt.tmp` for `a.txt`, then moved into place. A manifest is refused
|
||||||
downloaded, since saving the manifest would replace it.
|
before any file is downloaded if it lists a file where fetch writes
|
||||||
|
another: at the temp file of a listed file, or at `index.mf` or
|
||||||
|
`.index.mf.tmp` at the top of the tree. Names are compared in any letter
|
||||||
|
case.
|
||||||
- `mfer fetch --require-signature <fingerprint> https://example.com/stuff/`
|
- `mfer fetch --require-signature <fingerprint> https://example.com/stuff/`
|
||||||
- as above, but first refuses a manifest not signed by the key with that
|
- as above, but first refuses a manifest not signed by the key with that
|
||||||
fingerprint, as `mfer check --require-signature` does, before downloading
|
fingerprint, as `mfer check --require-signature` does, before downloading
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
// The developer tools this repo runs with `go tool`: gofumpt for
|
||||||
|
// script/gofumpt and protoc-gen-go for script/generate. Kept out of the mfer
|
||||||
|
// module so they add nothing to what mfer's users download. `go tool` builds
|
||||||
|
// exactly the source whose hashes go.sum here records.
|
||||||
|
module sneak.berlin/go/mfer/bin/tools
|
||||||
|
|
||||||
|
go 1.26.0
|
||||||
|
|
||||||
|
tool (
|
||||||
|
google.golang.org/protobuf/cmd/protoc-gen-go
|
||||||
|
mvdan.cc/gofumpt
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
|
golang.org/x/sync v0.22.0 // indirect
|
||||||
|
golang.org/x/tools v0.49.0 // indirect
|
||||||
|
// protoc-gen-go v1.36.11, 2026-10-04
|
||||||
|
google.golang.org/protobuf v1.36.11 // indirect
|
||||||
|
// gofumpt v0.12.0, 2026-10-04
|
||||||
|
mvdan.cc/gofumpt v0.12.0 // indirect
|
||||||
|
)
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
|
||||||
|
github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
|
||||||
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
|
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
|
||||||
|
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||||
|
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||||
|
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||||
|
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||||
|
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
|
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||||
|
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||||
|
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||||
|
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||||
|
mvdan.cc/gofumpt v0.12.0 h1:1Lbudkz2kpM9Cjz2pL4M19u7q+GaEhCTNf7N9mfpcho=
|
||||||
|
mvdan.cc/gofumpt v0.12.0/go.mod h1:SmBHHrljiZu/uoypeKup3rFzP6eoC9UwCp2iH5E3jZA=
|
||||||
+36
-18
@@ -91,10 +91,10 @@ var (
|
|||||||
// errHashMismatch indicates a downloaded file whose hash matches no
|
// errHashMismatch indicates a downloaded file whose hash matches no
|
||||||
// manifest hash.
|
// manifest hash.
|
||||||
errHashMismatch = errors.New("hash mismatch")
|
errHashMismatch = errors.New("hash mismatch")
|
||||||
// errManifestNameListed indicates a manifest that lists a file where
|
// errNameClash indicates a manifest that lists a file where fetch
|
||||||
// fetch saves the manifest.
|
// writes another file.
|
||||||
errManifestNameListed = errors.New(
|
errNameClash = errors.New(
|
||||||
"manifest lists a file where fetch saves the manifest")
|
"manifest lists a file where fetch writes another file")
|
||||||
)
|
)
|
||||||
|
|
||||||
// DownloadProgress reports the progress of a single file download.
|
// DownloadProgress reports the progress of a single file download.
|
||||||
@@ -412,7 +412,7 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
|
|||||||
|
|
||||||
// fetchManifest downloads the manifest at manifestURL and parses it,
|
// fetchManifest downloads the manifest at manifestURL and parses it,
|
||||||
// enforcing --require-signature if it is given and refusing a manifest
|
// enforcing --require-signature if it is given and refusing a manifest
|
||||||
// that lists a file where it will be saved. It returns the manifest as
|
// that lists a file where fetch writes another. It returns the manifest as
|
||||||
// downloaded, to be saved once the files are in place, and the files it
|
// downloaded, to be saved once the files are in place, and the files it
|
||||||
// lists.
|
// lists.
|
||||||
func fetchManifest(
|
func fetchManifest(
|
||||||
@@ -452,7 +452,7 @@ func fetchManifest(
|
|||||||
|
|
||||||
files := manifest.Files()
|
files := manifest.Files()
|
||||||
|
|
||||||
err = checkManifestNameUnlisted(files)
|
err = checkNoNameClash(files)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
@@ -462,19 +462,37 @@ func fetchManifest(
|
|||||||
return manifestData, files, nil
|
return manifestData, files, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkManifestNameUnlisted returns an error if files lists a file or
|
// checkNoNameClash returns an error if files lists a file, or a directory
|
||||||
// directory at the top of the tree under the name fetch saves the
|
// a file is in, under a name where fetch writes another file: the temp
|
||||||
// manifest as, or under that name's temp file. Saving the manifest would
|
// file it downloads a listed file to, or, at the top of the tree, the
|
||||||
// replace or remove it, or fail once every file was downloaded, leaving a
|
// saved manifest or its temp file. fetch would remove or replace what is
|
||||||
// tree check rejects. Names are compared ignoring case, since on a
|
// listed there, or fail partway, leaving a tree check rejects. Names are
|
||||||
// case-insensitive filesystem INDEX.MF and index.mf are one file.
|
// compared ignoring case, since on a case-insensitive filesystem INDEX.MF
|
||||||
func checkManifestNameUnlisted(files []*mfer.MFFilePath) error {
|
// and index.mf are one file.
|
||||||
for _, f := range files {
|
func checkNoNameClash(files []*mfer.MFFilePath) error {
|
||||||
top, _, _ := strings.Cut(filepath.Clean(f.GetPath()), string(filepath.Separator))
|
sep := string(filepath.Separator)
|
||||||
|
|
||||||
if strings.EqualFold(top, defaultManifestName) ||
|
// written maps each name fetch writes, other than the listed files
|
||||||
strings.EqualFold(top, tempPathFor(defaultManifestName)) {
|
// themselves, in lower case, to the file it writes there.
|
||||||
return fmt.Errorf("%w: %s", errManifestNameListed, f.GetPath())
|
written := map[string]string{
|
||||||
|
defaultManifestName: "the saved manifest",
|
||||||
|
tempPathFor(defaultManifestName): "the saved manifest's temp file",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range files {
|
||||||
|
tmpPath := tempPathFor(filepath.Clean(f.GetPath()))
|
||||||
|
written[strings.ToLower(tmpPath)] = "the temp file for " + f.GetPath()
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range files {
|
||||||
|
// Look up each directory on the file's path, then the file itself.
|
||||||
|
parts := strings.Split(strings.ToLower(filepath.Clean(f.GetPath())), sep)
|
||||||
|
|
||||||
|
for i := range parts {
|
||||||
|
what, ok := written[strings.Join(parts[:i+1], sep)]
|
||||||
|
if ok {
|
||||||
|
return fmt.Errorf("%w: %s (%s)", errNameClash, f.GetPath(), what)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+78
-12
@@ -1176,23 +1176,89 @@ func TestFetchRefusesListedManifestName(t *testing.T) {
|
|||||||
t.Run(listed, func(t *testing.T) {
|
t.Run(listed, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Built directly rather than scanned, since a scan lists no
|
files := map[string][]byte{listed: []byte("listed")}
|
||||||
// hidden files and never a path starting with "./".
|
|
||||||
content := []byte("listed")
|
|
||||||
builder := mfer.NewBuilder()
|
|
||||||
_, err := builder.AddFile(mfer.RelFilePath(listed), mfer.FileSize(len(content)),
|
|
||||||
mfer.ModTime(time.Now()), bytes.NewReader(content), nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var manifest bytes.Buffer
|
assertFetchRefused(t, builtManifest(t, files), files,
|
||||||
require.NoError(t, builder.Build(context.Background(), &manifest))
|
"manifest lists a file where fetch writes another file: "+listed)
|
||||||
|
|
||||||
assertFetchRefused(t, manifest.Bytes(), map[string][]byte{listed: content},
|
|
||||||
"manifest lists a file where fetch saves the manifest: "+listed)
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestFetchRefusesListedTempName fetches manifests that list a.txt and
|
||||||
|
// .a.txt.tmp, the temp file fetch downloads a.txt to, at the top of the
|
||||||
|
// tree and in a directory. Downloading a.txt would remove .a.txt.tmp, so
|
||||||
|
// fetch must refuse the manifest before it creates the destination or
|
||||||
|
// requests any file. README with .README.tmp checks that temp names are
|
||||||
|
// compared ignoring case. A manifest that lists only one of the two is
|
||||||
|
// fetched in full.
|
||||||
|
func TestFetchRefusesListedTempName(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, dir := range []string{"", "sub/"} {
|
||||||
|
for file, tmp := range map[string]string{
|
||||||
|
dir + "a.txt": dir + ".a.txt.tmp",
|
||||||
|
dir + "README": dir + ".README.tmp",
|
||||||
|
} {
|
||||||
|
both := map[string][]byte{
|
||||||
|
file: []byte("a file"),
|
||||||
|
tmp: []byte("a file at its temp name"),
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run(file+" and "+tmp, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assertFetchRefused(t, builtManifest(t, both), both,
|
||||||
|
"manifest lists a file where fetch writes another file: "+
|
||||||
|
tmp+" (the temp file for "+file+")")
|
||||||
|
})
|
||||||
|
|
||||||
|
for listed, content := range both {
|
||||||
|
t.Run("only "+listed, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
files := map[string][]byte{listed: content}
|
||||||
|
manifest := builtManifest(t, files)
|
||||||
|
|
||||||
|
server := httptest.NewServer(fetchTestHandler(manifest, files))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
dest := t.TempDir()
|
||||||
|
|
||||||
|
opts := testOpts([]string{
|
||||||
|
testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL,
|
||||||
|
}, afero.NewOsFs())
|
||||||
|
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||||
|
|
||||||
|
want := maps.Clone(files)
|
||||||
|
want[defaultManifestName] = manifest
|
||||||
|
assert.Equal(t, want, filesUnder(t, dest))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// builtManifest returns a manifest of files, built directly rather than
|
||||||
|
// scanned, since a scan lists no hidden files and never a path starting
|
||||||
|
// with "./".
|
||||||
|
func builtManifest(t *testing.T, files map[string][]byte) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
builder := mfer.NewBuilder()
|
||||||
|
|
||||||
|
for p, content := range files {
|
||||||
|
_, err := builder.AddFile(mfer.RelFilePath(p), mfer.FileSize(len(content)),
|
||||||
|
mfer.ModTime(time.Now()), bytes.NewReader(content), nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var manifest bytes.Buffer
|
||||||
|
|
||||||
|
require.NoError(t, builder.Build(context.Background(), &manifest))
|
||||||
|
|
||||||
|
return manifest.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
// assertFetchRefused serves manifest, a manifest of files, and fetches it
|
// assertFetchRefused serves manifest, a manifest of files, and fetches it
|
||||||
// with flags into a directory that does not exist yet. fetch must fail
|
// with flags into a directory that does not exist yet. fetch must fail
|
||||||
// with message after requesting only the manifest, and must not create
|
// with message after requesting only the manifest, and must not create
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
package mfer
|
|
||||||
|
|
||||||
//go:generate protoc ./mf.proto --go_out=paths=source_relative:.
|
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
{
|
{
|
||||||
"name": "mfer",
|
"name": "mfer",
|
||||||
"version": "0.1.0",
|
|
||||||
"private": true,
|
"private": true,
|
||||||
"description": "Development tooling for the mfer repository: prettier, used by script/fmt and script/fmt-check to format and verify Markdown and JSON.",
|
"description": "Development tooling for the mfer repository: prettier, used by script/fmt and script/fmt-check to format and verify Markdown and JSON.",
|
||||||
"license": "WTFPL",
|
"license": "WTFPL",
|
||||||
|
|||||||
+59
-5
@@ -13,11 +13,15 @@ set -eu
|
|||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
# Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions.
|
# Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions.
|
||||||
NODE_VERSION="22.17.0"
|
# The node version is in .nvmrc, where script/prettier reads it too.
|
||||||
|
NODE_VERSION="$(cat "$ROOT/.nvmrc")"
|
||||||
NVM_VERSION="0.40.3"
|
NVM_VERSION="0.40.3"
|
||||||
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
|
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
|
||||||
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
|
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
|
||||||
YARN_VERSION="1.22.22"
|
YARN_VERSION="1.22.22"
|
||||||
|
# protoc v33.4, 2026-10-04, for script/generate. The sha256 of each
|
||||||
|
# platform's release archive is in ensure_protoc.
|
||||||
|
PROTOC_VERSION="33.4"
|
||||||
|
|
||||||
PKGMGR=""
|
PKGMGR=""
|
||||||
SUDO=""
|
SUDO=""
|
||||||
@@ -74,9 +78,11 @@ verify_sha256() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# nvm is a bash script; run a command in a bash with nvm loaded
|
# nvm is a bash script; run a command in a bash with nvm loaded.
|
||||||
|
# --no-use: otherwise loading nvm here switches to the version .nvmrc
|
||||||
|
# names, and fails silently while that version is not installed yet.
|
||||||
nvm_sh() {
|
nvm_sh() {
|
||||||
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
|
bash -c ". \"\$HOME/.nvm/nvm.sh\" --no-use && $*"
|
||||||
}
|
}
|
||||||
|
|
||||||
ensure_nvm() {
|
ensure_nvm() {
|
||||||
@@ -122,6 +128,48 @@ install_js_deps() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Unpack protoc's release archive for this platform into bin/protoc, after
|
||||||
|
# checking the archive's sha256, unless bin/protoc already holds the pinned
|
||||||
|
# version.
|
||||||
|
ensure_protoc() {
|
||||||
|
dir="$ROOT/bin/protoc"
|
||||||
|
if [ "$("$dir/bin/protoc" --version 2>/dev/null)" = \
|
||||||
|
"libprotoc $PROTOC_VERSION" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
case "$(uname -s) $(uname -m)" in
|
||||||
|
"Linux x86_64")
|
||||||
|
platform="linux-x86_64"
|
||||||
|
sha256="c0040ea9aef08fdeb2c74ca609b18d5fdbfc44ea0042fcfbfb38860d35f7dd66"
|
||||||
|
;;
|
||||||
|
"Linux aarch64" | "Linux arm64")
|
||||||
|
platform="linux-aarch_64"
|
||||||
|
sha256="15aa988f4a6090636525ec236a8e4b3aab41eef402751bd5bb2df6afd9b7b5a5"
|
||||||
|
;;
|
||||||
|
"Darwin x86_64")
|
||||||
|
platform="osx-x86_64"
|
||||||
|
sha256="a49bec10d039e902d3b43e49938c42526f90011467609864fa6386ac4014da58"
|
||||||
|
;;
|
||||||
|
"Darwin arm64")
|
||||||
|
platform="osx-aarch_64"
|
||||||
|
sha256="726297dcfed58592fd35620a5a6246ae020c39e88f3fd4cb1827df7bcf3dfcf1"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "bootstrap: no protoc archive pinned for $(uname -s) $(uname -m)" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
if missing curl; then pkg_install curl curl curl curl; fi
|
||||||
|
if missing unzip; then pkg_install unzip unzip unzip unzip; fi
|
||||||
|
tmp="$(mktemp -d)"
|
||||||
|
curl -fsSL -o "$tmp/protoc.zip" \
|
||||||
|
"https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip"
|
||||||
|
verify_sha256 "$tmp/protoc.zip" "$sha256"
|
||||||
|
rm -rf "$dir"
|
||||||
|
unzip -q "$tmp/protoc.zip" -d "$dir"
|
||||||
|
rm -rf "$tmp"
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
@@ -132,8 +180,10 @@ main() {
|
|||||||
# ---- JS / docs repos ----
|
# ---- JS / docs repos ----
|
||||||
# This is a Go repo, but node and yarn are required anyway: prettier
|
# This is a Go repo, but node and yarn are required anyway: prettier
|
||||||
# formats the Markdown and JSON, and script/fmt-check verifies it.
|
# formats the Markdown and JSON, and script/fmt-check verifies it.
|
||||||
# The version is pinned by package.json/yarn.lock, whose integrity
|
# The version is pinned by package.json/yarn.lock: yarn checks every
|
||||||
# hashes --frozen-lockfile enforces.
|
# package it fetches against its yarn.lock integrity hash, and
|
||||||
|
# --frozen-lockfile fails instead of rewriting a yarn.lock that no
|
||||||
|
# longer matches package.json.
|
||||||
ensure_node
|
ensure_node
|
||||||
ensure_yarn
|
ensure_yarn
|
||||||
install_js_deps
|
install_js_deps
|
||||||
@@ -142,6 +192,10 @@ main() {
|
|||||||
if missing go; then pkg_install go golang go go; fi
|
if missing go; then pkg_install go golang go go; fi
|
||||||
# No golangci-lint: script/lint runs it in Docker only.
|
# No golangci-lint: script/lint runs it in Docker only.
|
||||||
go mod download
|
go mod download
|
||||||
|
# gofumpt and protoc-gen-go: bin/tools/go.mod pins them, and
|
||||||
|
# script/gofumpt and script/generate build them from there.
|
||||||
|
(cd "$ROOT/bin/tools" && go mod download)
|
||||||
|
ensure_protoc
|
||||||
|
|
||||||
# ---- Python repos ----
|
# ---- Python repos ----
|
||||||
# if missing python3; then pkg_install python3 python3 python3 python3; fi
|
# if missing python3; then pkg_install python3 python3 python3 python3; fi
|
||||||
|
|||||||
+20
-21
@@ -4,26 +4,17 @@
|
|||||||
# regenerates mf.pb.go: it is committed, so building and checking need no
|
# regenerates mf.pb.go: it is committed, so building and checking need no
|
||||||
# protoc. A test fails while mf.proto no longer matches the recorded hash.
|
# protoc. A test fails while mf.proto no longer matches the recorded hash.
|
||||||
#
|
#
|
||||||
# Needs exactly the protoc and protoc-gen-go versions named in the header of
|
# Runs the protoc that script/bootstrap unpacks into bin/protoc, and the
|
||||||
# the committed mf.pb.go (README.md says how to install them). Another
|
# protoc-gen-go that bin/tools/go.mod pins. Another version of either
|
||||||
# version writes a different mf.pb.go, so the script refuses to run.
|
# writes a different mf.pb.go.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
# protoc 33.4 names itself v6.33.4 in the mf.pb.go header.
|
# The protoc version script/bootstrap installs. protoc 33.4 names itself
|
||||||
|
# v6.33.4 in the mf.pb.go header.
|
||||||
PROTOC_VERSION="33.4"
|
PROTOC_VERSION="33.4"
|
||||||
PROTOC_GEN_GO_VERSION="v1.36.11"
|
PROTOC="$ROOT/bin/protoc/bin/protoc"
|
||||||
|
|
||||||
# require_version <command> <its exact --version output>
|
|
||||||
require_version() {
|
|
||||||
actual="$("$1" --version 2>/dev/null || true)"
|
|
||||||
if [ "$actual" != "$2" ]; then
|
|
||||||
echo "generate: needs $2 on PATH, found: ${actual:-none}" >&2
|
|
||||||
echo " README.md says how to install it." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# sha256 <file>: print "<hash> <file>", with sha256sum, or with shasum
|
# sha256 <file>: print "<hash> <file>", with sha256sum, or with shasum
|
||||||
# where there is no sha256sum.
|
# where there is no sha256sum.
|
||||||
@@ -39,16 +30,24 @@ sha256() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
|
# A bin/protoc left from before the pin moved fails here, until
|
||||||
|
# script/bootstrap replaces it.
|
||||||
|
actual="$("$PROTOC" --version 2>/dev/null || true)"
|
||||||
|
if [ "$actual" != "libprotoc $PROTOC_VERSION" ]; then
|
||||||
|
echo "generate: needs protoc $PROTOC_VERSION in bin/protoc," \
|
||||||
|
"found: ${actual:-none}; run script/bootstrap" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# `go tool -n` builds protoc-gen-go from bin/tools and prints where the
|
||||||
|
# binary is, without running it.
|
||||||
|
plugin="$(cd "$ROOT/bin/tools" && go tool -n protoc-gen-go)"
|
||||||
|
|
||||||
cd "$ROOT/mfer"
|
cd "$ROOT/mfer"
|
||||||
# `go install` puts protoc-gen-go in $(go env GOPATH)/bin, which is
|
|
||||||
# often not on PATH.
|
|
||||||
PATH="$PATH:$(go env GOPATH)/bin"
|
|
||||||
require_version protoc "libprotoc $PROTOC_VERSION"
|
|
||||||
require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION"
|
|
||||||
# Hashed before regenerating, so a missing hash tool stops the script
|
# Hashed before regenerating, so a missing hash tool stops the script
|
||||||
# before it changes anything. Regenerating leaves mf.proto as it is.
|
# before it changes anything. Regenerating leaves mf.proto as it is.
|
||||||
proto_hash="$(sha256 mf.proto)"
|
proto_hash="$(sha256 mf.proto)"
|
||||||
go generate .
|
"$PROTOC" --plugin=protoc-gen-go="$plugin" \
|
||||||
|
--go_out=paths=source_relative:. ./mf.proto
|
||||||
echo "$proto_hash" >mf.proto.sha256
|
echo "$proto_hash" >mf.proto.sha256
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+9
-9
@@ -10,10 +10,10 @@ set -eu
|
|||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
# gofumpt v0.12.0, 2026-10-04. `go run` fetches and builds exactly this
|
# The gofumpt version is the one bin/tools/go.mod pins. `go tool` run in
|
||||||
# version, so neither a developer machine nor the lint image needs
|
# bin/tools builds it from source checked against the hashes in
|
||||||
# gofumpt installed.
|
# bin/tools/go.sum, so neither a developer machine nor the lint image needs
|
||||||
GOFUMPT="mvdan.cc/gofumpt@v0.12.0"
|
# it installed.
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
echo "usage: script/gofumpt --write|--check" >&2
|
echo "usage: script/gofumpt --write|--check" >&2
|
||||||
@@ -22,15 +22,15 @@ usage() {
|
|||||||
|
|
||||||
main() {
|
main() {
|
||||||
[ "$#" -eq 1 ] || usage
|
[ "$#" -eq 1 ] || usage
|
||||||
cd "$ROOT"
|
cd "$ROOT/bin/tools"
|
||||||
# Every Go file in the repo. gofumpt holds generated files, such as
|
# Every Go file in the repo, from $ROOT down. gofumpt holds generated
|
||||||
# mfer/mf.pb.go, to gofmt's rules only.
|
# files, such as mfer/mf.pb.go, to gofmt's rules only.
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--write) go run "$GOFUMPT" -l -w . ;;
|
--write) go tool gofumpt -l -w "$ROOT" ;;
|
||||||
--check)
|
--check)
|
||||||
# Own line: a failing command inside `[ -n "$(...)" ]` does
|
# Own line: a failing command inside `[ -n "$(...)" ]` does
|
||||||
# not trip `set -e`, so a gofumpt that never ran would pass.
|
# not trip `set -e`, so a gofumpt that never ran would pass.
|
||||||
unformatted="$(go run "$GOFUMPT" -l .)"
|
unformatted="$(go tool gofumpt -l "$ROOT")"
|
||||||
if [ -n "$unformatted" ]; then
|
if [ -n "$unformatted" ]; then
|
||||||
echo "gofumpt: files need formatting (run make fmt):" >&2
|
echo "gofumpt: files need formatting (run make fmt):" >&2
|
||||||
echo "$unformatted" >&2
|
echo "$unformatted" >&2
|
||||||
|
|||||||
+24
-24
@@ -18,24 +18,9 @@ usage() {
|
|||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
|
|
||||||
# Prefer the version pinned by package.json/yarn.lock so that CI and
|
# Only the prettier yarn installed from yarn.lock, never one on PATH: a
|
||||||
# developer machines format identically. Fall back to a prettier on PATH,
|
# different version formats differently.
|
||||||
# but say so, because a different version formats differently.
|
PRETTIER="$ROOT/node_modules/.bin/prettier"
|
||||||
find_prettier() {
|
|
||||||
if [ -x "$ROOT/node_modules/.bin/prettier" ]; then
|
|
||||||
printf '%s\n' "$ROOT/node_modules/.bin/prettier"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
if command -v prettier >/dev/null 2>&1; then
|
|
||||||
echo "prettier: node_modules/.bin/prettier is absent; using the" \
|
|
||||||
"prettier on PATH, which may be a different version than the" \
|
|
||||||
"one pinned in package.json. Run script/bootstrap to install" \
|
|
||||||
"the pinned version." >&2
|
|
||||||
command -v prettier
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
[ "$#" -eq 1 ] || usage
|
[ "$#" -eq 1 ] || usage
|
||||||
@@ -46,10 +31,26 @@ main() {
|
|||||||
|
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
if ! prettier_bin="$(find_prettier)"; then
|
# Where there is no node on PATH, script/bootstrap installs the version
|
||||||
echo "prettier: not found." >&2
|
# .nvmrc names through nvm, which keeps it in this directory.
|
||||||
echo " Install it with: script/bootstrap" >&2
|
if ! command -v node >/dev/null 2>&1; then
|
||||||
echo " (installs the version pinned in package.json/yarn.lock)" >&2
|
PATH="$HOME/.nvm/versions/node/v$(cat .nvmrc)/bin:$PATH"
|
||||||
|
fi
|
||||||
|
if ! command -v node >/dev/null 2>&1; then
|
||||||
|
echo "prettier: node is missing; run script/bootstrap" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# node_modules keeps the old prettier after package.json moves to a new
|
||||||
|
# one, until script/bootstrap runs again, so compare the two.
|
||||||
|
if ! installed="$("$PRETTIER" --version 2>/dev/null)"; then
|
||||||
|
echo "prettier: not installed; run script/bootstrap" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
pinned="$(node -p 'require("./package.json").devDependencies.prettier')"
|
||||||
|
if [ "$installed" != "$pinned" ]; then
|
||||||
|
echo "prettier: package.json pins $pinned but $installed is" \
|
||||||
|
"installed; run script/bootstrap" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -62,8 +63,7 @@ main() {
|
|||||||
# patterns always match at least one tracked file (README.md,
|
# patterns always match at least one tracked file (README.md,
|
||||||
# package.json), so an empty match means the glob broke, and prettier
|
# package.json), so an empty match means the glob broke, and prettier
|
||||||
# erroring out is exactly what we want rather than a vacuous pass.
|
# erroring out is exactly what we want rather than a vacuous pass.
|
||||||
"$prettier_bin" "$mode" "**/*.md"
|
"$PRETTIER" "$mode" "**/*.md" "**/*.json"
|
||||||
"$prettier_bin" "$mode" "**/*.json"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
Reference in New Issue
Block a user