1 Commits
Author SHA1 Message Date
sneak e786ff516e fetch: destination directory, skip files already present, save the manifest, require a signer (closes #101)
check / check (push) Failing after 3s
fetch takes --dest (default .) and writes every file there through the
existing symlink and hard-link guards, which now work relative to that
directory. A file already there with the listed size and hash is
skipped; a leftover temp file is still replaced. Once every file
verifies, the manifest is saved as index.mf through the same temp file
and rename, so check runs on the result. --require-signature is shared
with check and enforced through verifyRequiredSigner, on a Checker over
the manifest held in memory, before anything is downloaded or written.

Model: opus-5-5
2026-10-04 15:13:33 +00:00
9 changed files with 66 additions and 90 deletions

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
/bin/mfer
/bin/
/tmp
/node_modules/
*.tmp
+1 -1
View File
@@ -26,7 +26,7 @@ source for coding standards, formatting, linting, and workflow rules.
- This is a Go library + CLI tool for generating `.mf` manifest files.
- The proto definition is in `mfer/mf.proto`; generated `.pb.go` files are
committed (required for `go get` compatibility).
- The format specification is in `docs/FORMAT.md`.
- The format specification is in `FORMAT.md`.
- Open work, open design questions included, is tracked only in the repo's
issues: https://git.eeqj.de/sneak/mfer/issues. There is no `TODO.md` and no
TODO list in `README.md`; do not add either. For this repo this overrides the
+2 -2
View File
@@ -19,7 +19,7 @@ An `.mf` file consists of two parts, concatenated:
There is no length prefix or version byte between the magic and the protobuf
message. The protobuf message extends to the end of the file.
See [`mfer/mf.proto`](../mfer/mf.proto) for exact field numbers and types.
See [`mfer/mf.proto`](mfer/mf.proto) for exact field numbers and types.
## Outer Message (`MFFileOuter`)
@@ -143,6 +143,6 @@ The recommended MIME type for `.mf` files is `application/octet-stream`. The
## Reference
- Proto definition: [`mfer/mf.proto`](../mfer/mf.proto)
- Proto definition: [`mfer/mf.proto`](mfer/mf.proto)
- Reference implementation:
[git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
+1 -1
View File
@@ -244,7 +244,7 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
as question 10 on [issue 82](https://git.eeqj.de/sneak/mfer/issues/82).
- Should the on-disk serialization format be proto3 or json? Settled: it is
proto3, see `docs/FORMAT.md` and `mfer/mf.proto`.
proto3, see `FORMAT.md` and `mfer/mf.proto`.
# Tool Examples
+7 -12
View File
@@ -1,12 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
main() {
if [[ -n "${GITREV:-}" ]]; then
echo "$GITREV"
else
git describe --tags --always --dirty=-dirty
fi
}
main "$@"
#!/bin/bash
#
if [[ ! -z "$GITREV" ]]; then
echo $GITREV
else
git describe --tags --always --dirty=-dirty
fi
-23
View File
@@ -1,23 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# usage.sh - Generate and check a manifest from the repo
# Run from repo root: bin/usage.sh
cleanup() {
rm -rf "$TMPDIR"
}
main() {
TMPDIR=$(mktemp -d)
MANIFEST="$TMPDIR/index.mf"
trap cleanup EXIT
echo "Building mfer..."
go build -o "$TMPDIR/mfer" ./cmd/mfer
"$TMPDIR/mfer" generate -o "$MANIFEST" .
"$TMPDIR/mfer" check --base . "$MANIFEST"
}
main "$@"
+19
View File
@@ -0,0 +1,19 @@
#!/bin/bash
set -euo pipefail
# usage.sh - Generate and check a manifest from the repo
# Run from repo root: ./contrib/usage.sh
TMPDIR=$(mktemp -d)
MANIFEST="$TMPDIR/index.mf"
cleanup() {
rm -rf "$TMPDIR"
}
trap cleanup EXIT
echo "Building mfer..."
go build -o "$TMPDIR/mfer" ./cmd/mfer
"$TMPDIR/mfer" generate -o "$MANIFEST" .
"$TMPDIR/mfer" check --base . "$MANIFEST"
+34 -49
View File
@@ -346,11 +346,41 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
firstDelay: firstRetryDelay,
}
manifestData, files, err := fetchManifest(ctx, client, manifestURL)
log.Infof("fetching manifest from %s", manifestURL)
// Read the whole manifest before parsing it, so that a connection
// lost partway through is retried rather than reported as a bad
// manifest.
var manifestData []byte
err = client.get(ctx.Context, manifestURL, func(resp *http.Response) error {
var readErr error
manifestData, readErr = io.ReadAll(resp.Body)
return readErr
})
if err != nil {
return err
return fmt.Errorf("failed to fetch manifest: %w", err)
}
// Parse manifest
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
if err != nil {
return fmt.Errorf("failed to parse manifest: %w", err)
}
requiredSigner := ctx.String(flagRequireSignature)
if requiredSigner != "" {
err = verifyFetchedSigner(ctx.Context, manifestData, requiredSigner)
if err != nil {
return err
}
}
files := manifest.Files()
log.Infof("manifest contains %d files", len(files))
// Compute base URL (directory containing manifest)
baseURL, err := manifestBaseURL(manifestURL)
if err != nil {
@@ -406,57 +436,12 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
return nil
}
// fetchManifest downloads the manifest at manifestURL and parses it,
// enforcing --require-signature if it is given. It returns the manifest as
// downloaded, to be saved once the files are in place, and the files it
// lists.
func fetchManifest(
ctx *cli.Context, client retryingClient, manifestURL string,
) ([]byte, []*mfer.MFFilePath, error) {
log.Infof("fetching manifest from %s", manifestURL)
// Read the whole manifest before parsing it, so that a connection
// lost partway through is retried rather than reported as a bad
// manifest.
var manifestData []byte
err := client.get(ctx.Context, manifestURL, func(resp *http.Response) error {
var readErr error
manifestData, readErr = io.ReadAll(resp.Body)
return readErr
})
if err != nil {
return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err)
}
// Parse manifest
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
if err != nil {
return nil, nil, fmt.Errorf("failed to parse manifest: %w", err)
}
requiredSigner := ctx.String(flagRequireSignature)
if requiredSigner != "" {
err = verifyFetchedSigner(ctx, manifestData, requiredSigner)
if err != nil {
return nil, nil, err
}
}
files := manifest.Files()
log.Infof("manifest contains %d files", len(files))
return manifestData, files, nil
}
// verifyFetchedSigner enforces --require-signature on the fetched manifest
// exactly as check does. verifyRequiredSigner takes a Checker, which loads
// its manifest from a file, so the manifest is handed to it as a file in
// memory.
func verifyFetchedSigner(
ctx *cli.Context, manifestData []byte, requiredSigner string,
ctx context.Context, manifestData []byte, requiredSigner string,
) error {
memFs := afero.NewMemMapFs()
manifestPath := "/" + defaultManifestName
@@ -475,7 +460,7 @@ func verifyFetchedSigner(
return fmt.Errorf("failed to load manifest: %w", err)
}
return verifyRequiredSigner(ctx.Context, chk, requiredSigner)
return verifyRequiredSigner(ctx, chk, requiredSigner)
}
// saveManifest writes the fetched manifest into dest under the default
+1 -1
View File
@@ -146,7 +146,7 @@ func commonFlags() []cli.Flag {
// requireSignatureFlag returns the --require-signature flag taken by the
// check and fetch subcommands.
func requireSignatureFlag() *cli.StringFlag {
func requireSignatureFlag() cli.Flag {
return &cli.StringFlag{
Name: flagRequireSignature,
Aliases: []string{"S"},