Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
438b73eddf | ||
|
|
400a2f8f63 |
No files matched your search
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
/bin/
|
/bin/mfer
|
||||||
/tmp
|
/tmp
|
||||||
/node_modules/
|
/node_modules/
|
||||||
*.tmp
|
*.tmp
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ source for coding standards, formatting, linting, and workflow rules.
|
|||||||
- This is a Go library + CLI tool for generating `.mf` manifest files.
|
- This is a Go library + CLI tool for generating `.mf` manifest files.
|
||||||
- The proto definition is in `mfer/mf.proto`; generated `.pb.go` files are
|
- The proto definition is in `mfer/mf.proto`; generated `.pb.go` files are
|
||||||
committed (required for `go get` compatibility).
|
committed (required for `go get` compatibility).
|
||||||
- The format specification is in `FORMAT.md`.
|
- The format specification is in `docs/FORMAT.md`.
|
||||||
- Open work, open design questions included, is tracked only in the repo's
|
- Open work, open design questions included, is tracked only in the repo's
|
||||||
issues: https://git.eeqj.de/sneak/mfer/issues. There is no `TODO.md` and no
|
issues: https://git.eeqj.de/sneak/mfer/issues. There is no `TODO.md` and no
|
||||||
TODO list in `README.md`; do not add either. For this repo this overrides the
|
TODO list in `README.md`; do not add either. For this repo this overrides the
|
||||||
|
|||||||
@@ -244,7 +244,7 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
|||||||
as question 10 on [issue 82](https://git.eeqj.de/sneak/mfer/issues/82).
|
as question 10 on [issue 82](https://git.eeqj.de/sneak/mfer/issues/82).
|
||||||
|
|
||||||
- Should the on-disk serialization format be proto3 or json? Settled: it is
|
- Should the on-disk serialization format be proto3 or json? Settled: it is
|
||||||
proto3, see `FORMAT.md` and `mfer/mf.proto`.
|
proto3, see `docs/FORMAT.md` and `mfer/mf.proto`.
|
||||||
|
|
||||||
# Tool Examples
|
# Tool Examples
|
||||||
|
|
||||||
|
|||||||
+12
-7
@@ -1,7 +1,12 @@
|
|||||||
#!/bin/bash
|
#!/usr/bin/env bash
|
||||||
#
|
set -euo pipefail
|
||||||
if [[ ! -z "$GITREV" ]]; then
|
|
||||||
echo $GITREV
|
main() {
|
||||||
else
|
if [[ -n "${GITREV:-}" ]]; then
|
||||||
git describe --tags --always --dirty=-dirty
|
echo "$GITREV"
|
||||||
fi
|
else
|
||||||
|
git describe --tags --always --dirty=-dirty
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Executable
+23
@@ -0,0 +1,23 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# usage.sh - Generate and check a manifest from the repo
|
||||||
|
# Run from repo root: bin/usage.sh
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
rm -rf "$TMPDIR"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
TMPDIR=$(mktemp -d)
|
||||||
|
MANIFEST="$TMPDIR/index.mf"
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
echo "Building mfer..."
|
||||||
|
go build -o "$TMPDIR/mfer" ./cmd/mfer
|
||||||
|
|
||||||
|
"$TMPDIR/mfer" generate -o "$MANIFEST" .
|
||||||
|
"$TMPDIR/mfer" check --base . "$MANIFEST"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# usage.sh - Generate and check a manifest from the repo
|
|
||||||
# Run from repo root: ./contrib/usage.sh
|
|
||||||
|
|
||||||
TMPDIR=$(mktemp -d)
|
|
||||||
MANIFEST="$TMPDIR/index.mf"
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
rm -rf "$TMPDIR"
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
echo "Building mfer..."
|
|
||||||
go build -o "$TMPDIR/mfer" ./cmd/mfer
|
|
||||||
|
|
||||||
"$TMPDIR/mfer" generate -o "$MANIFEST" .
|
|
||||||
"$TMPDIR/mfer" check --base . "$MANIFEST"
|
|
||||||
@@ -19,7 +19,7 @@ An `.mf` file consists of two parts, concatenated:
|
|||||||
There is no length prefix or version byte between the magic and the protobuf
|
There is no length prefix or version byte between the magic and the protobuf
|
||||||
message. The protobuf message extends to the end of the file.
|
message. The protobuf message extends to the end of the file.
|
||||||
|
|
||||||
See [`mfer/mf.proto`](mfer/mf.proto) for exact field numbers and types.
|
See [`mfer/mf.proto`](../mfer/mf.proto) for exact field numbers and types.
|
||||||
|
|
||||||
## Outer Message (`MFFileOuter`)
|
## Outer Message (`MFFileOuter`)
|
||||||
|
|
||||||
@@ -143,6 +143,6 @@ The recommended MIME type for `.mf` files is `application/octet-stream`. The
|
|||||||
|
|
||||||
## Reference
|
## Reference
|
||||||
|
|
||||||
- Proto definition: [`mfer/mf.proto`](mfer/mf.proto)
|
- Proto definition: [`mfer/mf.proto`](../mfer/mf.proto)
|
||||||
- Reference implementation:
|
- Reference implementation:
|
||||||
[git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
|
[git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
|
||||||
+49
-34
@@ -346,41 +346,11 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
|
|||||||
firstDelay: firstRetryDelay,
|
firstDelay: firstRetryDelay,
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("fetching manifest from %s", manifestURL)
|
manifestData, files, err := fetchManifest(ctx, client, manifestURL)
|
||||||
|
|
||||||
// Read the whole manifest before parsing it, so that a connection
|
|
||||||
// lost partway through is retried rather than reported as a bad
|
|
||||||
// manifest.
|
|
||||||
var manifestData []byte
|
|
||||||
|
|
||||||
err = client.get(ctx.Context, manifestURL, func(resp *http.Response) error {
|
|
||||||
var readErr error
|
|
||||||
|
|
||||||
manifestData, readErr = io.ReadAll(resp.Body)
|
|
||||||
|
|
||||||
return readErr
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to fetch manifest: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse manifest
|
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to parse manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
requiredSigner := ctx.String(flagRequireSignature)
|
|
||||||
if requiredSigner != "" {
|
|
||||||
err = verifyFetchedSigner(ctx.Context, manifestData, requiredSigner)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
files := manifest.Files()
|
|
||||||
log.Infof("manifest contains %d files", len(files))
|
|
||||||
|
|
||||||
// Compute base URL (directory containing manifest)
|
// Compute base URL (directory containing manifest)
|
||||||
baseURL, err := manifestBaseURL(manifestURL)
|
baseURL, err := manifestBaseURL(manifestURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -436,12 +406,57 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fetchManifest downloads the manifest at manifestURL and parses it,
|
||||||
|
// enforcing --require-signature if it is given. It returns the manifest as
|
||||||
|
// downloaded, to be saved once the files are in place, and the files it
|
||||||
|
// lists.
|
||||||
|
func fetchManifest(
|
||||||
|
ctx *cli.Context, client retryingClient, manifestURL string,
|
||||||
|
) ([]byte, []*mfer.MFFilePath, error) {
|
||||||
|
log.Infof("fetching manifest from %s", manifestURL)
|
||||||
|
|
||||||
|
// Read the whole manifest before parsing it, so that a connection
|
||||||
|
// lost partway through is retried rather than reported as a bad
|
||||||
|
// manifest.
|
||||||
|
var manifestData []byte
|
||||||
|
|
||||||
|
err := client.get(ctx.Context, manifestURL, func(resp *http.Response) error {
|
||||||
|
var readErr error
|
||||||
|
|
||||||
|
manifestData, readErr = io.ReadAll(resp.Body)
|
||||||
|
|
||||||
|
return readErr
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse manifest
|
||||||
|
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to parse manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
requiredSigner := ctx.String(flagRequireSignature)
|
||||||
|
if requiredSigner != "" {
|
||||||
|
err = verifyFetchedSigner(ctx, manifestData, requiredSigner)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
files := manifest.Files()
|
||||||
|
log.Infof("manifest contains %d files", len(files))
|
||||||
|
|
||||||
|
return manifestData, files, nil
|
||||||
|
}
|
||||||
|
|
||||||
// verifyFetchedSigner enforces --require-signature on the fetched manifest
|
// verifyFetchedSigner enforces --require-signature on the fetched manifest
|
||||||
// exactly as check does. verifyRequiredSigner takes a Checker, which loads
|
// exactly as check does. verifyRequiredSigner takes a Checker, which loads
|
||||||
// its manifest from a file, so the manifest is handed to it as a file in
|
// its manifest from a file, so the manifest is handed to it as a file in
|
||||||
// memory.
|
// memory.
|
||||||
func verifyFetchedSigner(
|
func verifyFetchedSigner(
|
||||||
ctx context.Context, manifestData []byte, requiredSigner string,
|
ctx *cli.Context, manifestData []byte, requiredSigner string,
|
||||||
) error {
|
) error {
|
||||||
memFs := afero.NewMemMapFs()
|
memFs := afero.NewMemMapFs()
|
||||||
manifestPath := "/" + defaultManifestName
|
manifestPath := "/" + defaultManifestName
|
||||||
@@ -460,7 +475,7 @@ func verifyFetchedSigner(
|
|||||||
return fmt.Errorf("failed to load manifest: %w", err)
|
return fmt.Errorf("failed to load manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return verifyRequiredSigner(ctx, chk, requiredSigner)
|
return verifyRequiredSigner(ctx.Context, chk, requiredSigner)
|
||||||
}
|
}
|
||||||
|
|
||||||
// saveManifest writes the fetched manifest into dest under the default
|
// saveManifest writes the fetched manifest into dest under the default
|
||||||
|
|||||||
@@ -146,7 +146,7 @@ func commonFlags() []cli.Flag {
|
|||||||
|
|
||||||
// requireSignatureFlag returns the --require-signature flag taken by the
|
// requireSignatureFlag returns the --require-signature flag taken by the
|
||||||
// check and fetch subcommands.
|
// check and fetch subcommands.
|
||||||
func requireSignatureFlag() cli.Flag {
|
func requireSignatureFlag() *cli.StringFlag {
|
||||||
return &cli.StringFlag{
|
return &cli.StringFlag{
|
||||||
Name: flagRequireSignature,
|
Name: flagRequireSignature,
|
||||||
Aliases: []string{"S"},
|
Aliases: []string{"S"},
|
||||||
|
|||||||
Reference in new issue
Block a user