Compare commits
2
Commits
7122b56a51
...
08139beab2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
08139beab2 | ||
|
|
d3394bd2a2 |
No files matched your search
+2
-3
@@ -59,6 +59,5 @@
|
||||
**/.vscode
|
||||
**/*.sublime-*
|
||||
|
||||
# This repo's own host-built archives (Makefile).
|
||||
*.tmp
|
||||
*.dockerimage
|
||||
# This repo's own host-built binary (make build).
|
||||
/bin/mfer
|
||||
@@ -1,11 +1,6 @@
|
||||
/bin/mfer
|
||||
/tmp
|
||||
/node_modules/
|
||||
*.tmp
|
||||
*.dockerimage
|
||||
/vendor
|
||||
vendor.tzst
|
||||
modcache.tzst
|
||||
|
||||
# Generated manifest files
|
||||
/index.mf
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ COPY . .
|
||||
# Go half of fmt-check only: this image has no node, so no prettier. The
|
||||
# markdown half runs in the mdfmt stage below. The image has no gofumpt
|
||||
# either; script/gofumpt builds the version it pins with `go run`.
|
||||
RUN make fmt-check-go
|
||||
RUN script/gofumpt --check
|
||||
# The linter directly, not `make lint`: script/lint builds this stage, and
|
||||
# there is no docker inside this build.
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
@@ -1,20 +1,8 @@
|
||||
export DOCKER_BUILDKIT := 1
|
||||
export PROGRESS_NO_TRUNC := 1
|
||||
GOPATH := $(shell go env GOPATH)
|
||||
export PATH := $(PATH):$(GOPATH)/bin
|
||||
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
|
||||
ARCH := $(shell uname -m)
|
||||
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown)
|
||||
APPNAME := mfer
|
||||
VERSION := 0.1.0
|
||||
export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
|
||||
GOLDFLAGS += -X main.Version=$(VERSION)
|
||||
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
|
||||
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
||||
.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks build generate fuzz
|
||||
|
||||
.PHONY: bootstrap setup docker default run ci test fuzz check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme generate
|
||||
|
||||
default: fmt test
|
||||
# Makefile targets are thin shims; the implementations live in script/
|
||||
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
|
||||
# of README.md).
|
||||
|
||||
bootstrap:
|
||||
@script/bootstrap
|
||||
@@ -22,75 +10,32 @@ bootstrap:
|
||||
setup:
|
||||
@script/setup
|
||||
|
||||
run: ./bin/mfer
|
||||
./$<
|
||||
./$< gen
|
||||
|
||||
ci: test
|
||||
|
||||
test:
|
||||
@script/test
|
||||
|
||||
fuzz:
|
||||
@script/fuzz
|
||||
|
||||
fixme:
|
||||
@grep -nir fixme . | grep -v Makefile
|
||||
|
||||
check:
|
||||
@script/check
|
||||
|
||||
fmt-check:
|
||||
@script/fmt-check
|
||||
|
||||
# Halves of fmt-check, for environments that have only one toolchain:
|
||||
# the Docker lint stage has Go but no node, the markdown stage the reverse.
|
||||
fmt-check-go:
|
||||
@script/gofumpt --check
|
||||
|
||||
fmt-check-md:
|
||||
@script/prettier --check
|
||||
|
||||
hooks:
|
||||
@script/install-precommit
|
||||
|
||||
generate:
|
||||
@script/generate
|
||||
|
||||
bin/mfer: $(SOURCEFILES)
|
||||
cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../bin/mfer $(GOFLAGS) .
|
||||
|
||||
clean:
|
||||
rm -rfv bin/mfer cmd/mfer/mfer *.dockerimage
|
||||
|
||||
fmt:
|
||||
@script/fmt
|
||||
|
||||
lint:
|
||||
@script/lint
|
||||
|
||||
fmt:
|
||||
@script/fmt
|
||||
|
||||
fmt-check:
|
||||
@script/fmt-check
|
||||
|
||||
check:
|
||||
@script/check
|
||||
|
||||
docker:
|
||||
@script/docker
|
||||
|
||||
sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
|
||||
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
|
||||
docker save sneak/mfer | pv | zstdmt -19 > $@
|
||||
du -sh $@
|
||||
hooks:
|
||||
@script/install-precommit
|
||||
|
||||
godoc:
|
||||
open http://127.0.0.1:6060
|
||||
godoc -http=:6060
|
||||
build:
|
||||
@script/build
|
||||
|
||||
vendor.tzst: go.mod go.sum
|
||||
go mod tidy
|
||||
go mod vendor
|
||||
cd vendor && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
|
||||
rm -rf vendor
|
||||
mv $@.tmp $@
|
||||
generate:
|
||||
@script/generate
|
||||
|
||||
modcache.tzst: go.mod go.sum
|
||||
go mod tidy
|
||||
cd $(HOME)/go/pkg && chmod -R u+rw . && rm -rf mod sumdb
|
||||
go mod download -x
|
||||
cd $(shell go env GOMODCACHE) && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
|
||||
mv $@.tmp $@
|
||||
fuzz:
|
||||
@script/fuzz
|
||||
@@ -29,7 +29,7 @@ is committed, so no `protoc` toolchain is required:
|
||||
```sh
|
||||
git clone https://git.eeqj.de/sneak/mfer.git
|
||||
cd mfer
|
||||
go build -o bin/mfer ./cmd/mfer
|
||||
make build
|
||||
```
|
||||
|
||||
Generate a manifest for a directory tree, verify it later, and fetch a published
|
||||
@@ -76,6 +76,9 @@ provide:
|
||||
such as `script/docker`
|
||||
- `script/test` — run the test suite (`go test`); one test fails when
|
||||
`mfer/mf.proto` no longer matches the hash `script/generate` recorded
|
||||
- `script/build` (`make build`) — build the `mfer` binary into `bin/mfer`,
|
||||
stamped with the revision `mfer version` prints: the output of
|
||||
`git describe --tags --always --dirty`, as `script/docker` passes it
|
||||
- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from
|
||||
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in
|
||||
`mfer/mf.proto.sha256`; the only thing that regenerates the committed
|
||||
@@ -85,8 +88,8 @@ provide:
|
||||
[its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4)
|
||||
and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11
|
||||
(`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which
|
||||
installs it in `$(go env GOPATH)/bin`; `make generate` adds that directory to
|
||||
`PATH`)
|
||||
installs it in `$(go env GOPATH)/bin`; `script/generate` adds that directory
|
||||
to `PATH`)
|
||||
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
|
||||
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
|
||||
as ordinary tests
|
||||
@@ -98,8 +101,8 @@ provide:
|
||||
- `script/gofumpt` — run `gofumpt` over every Go file in the repository in the
|
||||
given mode, `--write` or `--check`, at the one version it pins (built on
|
||||
demand by `go run`, so nothing installs it); `script/fmt`, `script/fmt-check`
|
||||
and the Docker lint stage (`make fmt-check-go`) all go through it, so they
|
||||
cannot disagree about Go formatting
|
||||
and the Docker lint stage all go through it, so they cannot disagree about Go
|
||||
formatting
|
||||
- `script/prettier` — run prettier over the repository's canonical file set
|
||||
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
||||
`--check`; the single definition of that file set, so `script/fmt` and
|
||||
@@ -265,9 +268,12 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
||||
cryptographic integrity of downloaded files. A file already there with the
|
||||
size and hash the manifest lists is skipped. Once every file is in place,
|
||||
the manifest is saved there as `index.mf`, so `mfer check` can verify the
|
||||
tree later. A manifest that lists `index.mf` (in any letter case) or
|
||||
`.index.mf.tmp` at the top of the tree is refused before any file is
|
||||
downloaded, since saving the manifest would replace it.
|
||||
tree later. Each file is downloaded to a temp file beside it, such as
|
||||
`.a.txt.tmp` for `a.txt`, then moved into place. A manifest is refused
|
||||
before any file is downloaded if it lists a file where fetch writes
|
||||
another: at the temp file of a listed file, or at `index.mf` or
|
||||
`.index.mf.tmp` at the top of the tree. Names are compared in any letter
|
||||
case.
|
||||
- `mfer fetch --require-signature <fingerprint> https://example.com/stuff/`
|
||||
- as above, but first refuses a manifest not signed by the key with that
|
||||
fingerprint, as `mfer check --require-signature` does, before downloading
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
main() {
|
||||
if [[ -n "${GITREV:-}" ]]; then
|
||||
echo "$GITREV"
|
||||
else
|
||||
git describe --tags --always --dirty=-dirty
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+36
-18
@@ -91,10 +91,10 @@ var (
|
||||
// errHashMismatch indicates a downloaded file whose hash matches no
|
||||
// manifest hash.
|
||||
errHashMismatch = errors.New("hash mismatch")
|
||||
// errManifestNameListed indicates a manifest that lists a file where
|
||||
// fetch saves the manifest.
|
||||
errManifestNameListed = errors.New(
|
||||
"manifest lists a file where fetch saves the manifest")
|
||||
// errNameClash indicates a manifest that lists a file where fetch
|
||||
// writes another file.
|
||||
errNameClash = errors.New(
|
||||
"manifest lists a file where fetch writes another file")
|
||||
)
|
||||
|
||||
// DownloadProgress reports the progress of a single file download.
|
||||
@@ -412,7 +412,7 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
|
||||
|
||||
// fetchManifest downloads the manifest at manifestURL and parses it,
|
||||
// enforcing --require-signature if it is given and refusing a manifest
|
||||
// that lists a file where it will be saved. It returns the manifest as
|
||||
// that lists a file where fetch writes another. It returns the manifest as
|
||||
// downloaded, to be saved once the files are in place, and the files it
|
||||
// lists.
|
||||
func fetchManifest(
|
||||
@@ -452,7 +452,7 @@ func fetchManifest(
|
||||
|
||||
files := manifest.Files()
|
||||
|
||||
err = checkManifestNameUnlisted(files)
|
||||
err = checkNoNameClash(files)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -462,19 +462,37 @@ func fetchManifest(
|
||||
return manifestData, files, nil
|
||||
}
|
||||
|
||||
// checkManifestNameUnlisted returns an error if files lists a file or
|
||||
// directory at the top of the tree under the name fetch saves the
|
||||
// manifest as, or under that name's temp file. Saving the manifest would
|
||||
// replace or remove it, or fail once every file was downloaded, leaving a
|
||||
// tree check rejects. Names are compared ignoring case, since on a
|
||||
// case-insensitive filesystem INDEX.MF and index.mf are one file.
|
||||
func checkManifestNameUnlisted(files []*mfer.MFFilePath) error {
|
||||
for _, f := range files {
|
||||
top, _, _ := strings.Cut(filepath.Clean(f.GetPath()), string(filepath.Separator))
|
||||
// checkNoNameClash returns an error if files lists a file, or a directory
|
||||
// a file is in, under a name where fetch writes another file: the temp
|
||||
// file it downloads a listed file to, or, at the top of the tree, the
|
||||
// saved manifest or its temp file. fetch would remove or replace what is
|
||||
// listed there, or fail partway, leaving a tree check rejects. Names are
|
||||
// compared ignoring case, since on a case-insensitive filesystem INDEX.MF
|
||||
// and index.mf are one file.
|
||||
func checkNoNameClash(files []*mfer.MFFilePath) error {
|
||||
sep := string(filepath.Separator)
|
||||
|
||||
if strings.EqualFold(top, defaultManifestName) ||
|
||||
strings.EqualFold(top, tempPathFor(defaultManifestName)) {
|
||||
return fmt.Errorf("%w: %s", errManifestNameListed, f.GetPath())
|
||||
// written maps each name fetch writes, other than the listed files
|
||||
// themselves, in lower case, to the file it writes there.
|
||||
written := map[string]string{
|
||||
defaultManifestName: "the saved manifest",
|
||||
tempPathFor(defaultManifestName): "the saved manifest's temp file",
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
tmpPath := tempPathFor(filepath.Clean(f.GetPath()))
|
||||
written[strings.ToLower(tmpPath)] = "the temp file for " + f.GetPath()
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
// Look up each directory on the file's path, then the file itself.
|
||||
parts := strings.Split(strings.ToLower(filepath.Clean(f.GetPath())), sep)
|
||||
|
||||
for i := range parts {
|
||||
what, ok := written[strings.Join(parts[:i+1], sep)]
|
||||
if ok {
|
||||
return fmt.Errorf("%w: %s (%s)", errNameClash, f.GetPath(), what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+73
-12
@@ -1176,23 +1176,84 @@ func TestFetchRefusesListedManifestName(t *testing.T) {
|
||||
t.Run(listed, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Built directly rather than scanned, since a scan lists no
|
||||
// hidden files and never a path starting with "./".
|
||||
content := []byte("listed")
|
||||
builder := mfer.NewBuilder()
|
||||
_, err := builder.AddFile(mfer.RelFilePath(listed), mfer.FileSize(len(content)),
|
||||
mfer.ModTime(time.Now()), bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
files := map[string][]byte{listed: []byte("listed")}
|
||||
|
||||
var manifest bytes.Buffer
|
||||
require.NoError(t, builder.Build(context.Background(), &manifest))
|
||||
|
||||
assertFetchRefused(t, manifest.Bytes(), map[string][]byte{listed: content},
|
||||
"manifest lists a file where fetch saves the manifest: "+listed)
|
||||
assertFetchRefused(t, builtManifest(t, files), files,
|
||||
"manifest lists a file where fetch writes another file: "+listed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchRefusesListedTempName fetches manifests that list a.txt and
|
||||
// .a.txt.tmp, the temp file fetch downloads a.txt to, at the top of the
|
||||
// tree and in a directory. Downloading a.txt would remove .a.txt.tmp, so
|
||||
// fetch must refuse the manifest before it creates the destination or
|
||||
// requests any file. A manifest that lists only one of the two is fetched
|
||||
// in full.
|
||||
func TestFetchRefusesListedTempName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, dir := range []string{"", "sub/"} {
|
||||
file, tmp := dir+"a.txt", dir+".a.txt.tmp"
|
||||
both := map[string][]byte{
|
||||
file: []byte("a file"),
|
||||
tmp: []byte("a file at its temp name"),
|
||||
}
|
||||
|
||||
t.Run(file+" and "+tmp, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assertFetchRefused(t, builtManifest(t, both), both,
|
||||
"manifest lists a file where fetch writes another file: "+
|
||||
tmp+" (the temp file for "+file+")")
|
||||
})
|
||||
|
||||
for listed, content := range both {
|
||||
t.Run("only "+listed, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
files := map[string][]byte{listed: content}
|
||||
manifest := builtManifest(t, files)
|
||||
|
||||
server := httptest.NewServer(fetchTestHandler(manifest, files))
|
||||
defer server.Close()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL,
|
||||
}, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
want := maps.Clone(files)
|
||||
want[defaultManifestName] = manifest
|
||||
assert.Equal(t, want, filesUnder(t, dest))
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// builtManifest returns a manifest of files, built directly rather than
|
||||
// scanned, since a scan lists no hidden files and never a path starting
|
||||
// with "./".
|
||||
func builtManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
builder := mfer.NewBuilder()
|
||||
|
||||
for p, content := range files {
|
||||
_, err := builder.AddFile(mfer.RelFilePath(p), mfer.FileSize(len(content)),
|
||||
mfer.ModTime(time.Now()), bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
var manifest bytes.Buffer
|
||||
|
||||
require.NoError(t, builder.Build(context.Background(), &manifest))
|
||||
|
||||
return manifest.Bytes()
|
||||
}
|
||||
|
||||
// assertFetchRefused serves manifest, a manifest of files, and fetches it
|
||||
// with flags into a directory that does not exist yet. fetch must fail
|
||||
// with message after requesting only the manifest, and must not create
|
||||
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
# script/build: build the mfer binary into bin/mfer, stamped with the
|
||||
# revision `mfer version` prints, derived as script/docker derives it.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
# Own line: a failing command substitution inside an argument does
|
||||
# not trip `set -e`, so the inline form degrades silently to an
|
||||
# empty constant.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
go build -tags urfave_cli_no_docs \
|
||||
-ldflags "-X main.Gitrev=$version" \
|
||||
-o bin/mfer ./cmd/mfer
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -40,6 +40,9 @@ sha256() {
|
||||
|
||||
main() {
|
||||
cd "$ROOT/mfer"
|
||||
# `go install` puts protoc-gen-go in $(go env GOPATH)/bin, which is
|
||||
# often not on PATH.
|
||||
PATH="$PATH:$(go env GOPATH)/bin"
|
||||
require_version protoc "libprotoc $PROTOC_VERSION"
|
||||
require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION"
|
||||
# Hashed before regenerating, so a missing hash tool stops the script
|
||||
|
||||
+2
-2
@@ -4,8 +4,8 @@
|
||||
# Takes exactly one mode argument, --write or --check, and runs the same
|
||||
# gofumpt version over the same files in both modes. script/fmt and
|
||||
# script/fmt-check both go through here, and so does the Docker lint
|
||||
# stage (make fmt-check-go), so what gets formatted and what gets
|
||||
# verified cannot drift apart.
|
||||
# stage, so what gets formatted and what gets verified cannot drift
|
||||
# apart.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
Reference in new issue
Block a user