Configure prettier and make fmt-check cover markdown (closes #69) #88
@@ -1,3 +1,4 @@
|
|||||||
*.tmp
|
*.tmp
|
||||||
*.dockerimage
|
*.dockerimage
|
||||||
.git
|
.git
|
||||||
|
node_modules
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,5 +1,6 @@
|
|||||||
/bin/
|
/bin/
|
||||||
/tmp
|
/tmp
|
||||||
|
/node_modules/
|
||||||
*.tmp
|
*.tmp
|
||||||
*.dockerimage
|
*.dockerimage
|
||||||
/vendor
|
/vendor
|
||||||
|
|||||||
14
.prettierignore
Normal file
14
.prettierignore
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
# REPO_POLICIES.md is a verbatim copy of an authoritative upstream
|
||||||
|
# document (sneak/prompts). Local tooling must never rewrite it: any
|
||||||
|
# reformatting is silent drift from the source of truth.
|
||||||
|
REPO_POLICIES.md
|
||||||
|
|
||||||
|
# User-owned configuration, copied verbatim from upstream. Not matched by
|
||||||
|
# the current prettier file set, but listed so widening that set can never
|
||||||
|
# start rewriting it.
|
||||||
|
.golangci.yml
|
||||||
|
|
||||||
|
# Dependencies and build output
|
||||||
|
node_modules/
|
||||||
|
vendor/
|
||||||
|
bin/
|
||||||
4
.prettierrc
Normal file
4
.prettierrc
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"tabWidth": 4,
|
||||||
|
"proseWrap": "always"
|
||||||
|
}
|
||||||
@@ -13,7 +13,8 @@ source for coding standards, formatting, linting, and workflow rules.
|
|||||||
- After each change, run `make fmt`, then `make test`, then `make lint`. Fix any
|
- After each change, run `make fmt`, then `make test`, then `make lint`. Fix any
|
||||||
failures before committing.
|
failures before committing.
|
||||||
|
|
||||||
- After each change, commit only the files you've changed. Push after committing.
|
- After each change, commit only the files you've changed. Push after
|
||||||
|
committing.
|
||||||
|
|
||||||
## Attribution
|
## Attribution
|
||||||
|
|
||||||
@@ -26,5 +27,5 @@ source for coding standards, formatting, linting, and workflow rules.
|
|||||||
- The proto definition is in `mfer/mf.proto`; generated `.pb.go` files are
|
- The proto definition is in `mfer/mf.proto`; generated `.pb.go` files are
|
||||||
committed (required for `go get` compatibility).
|
committed (required for `go get` compatibility).
|
||||||
- The format specification is in `FORMAT.md`.
|
- The format specification is in `FORMAT.md`.
|
||||||
- See the TODO section in `README.md` for the 1.0 implementation plan
|
- See the TODO section in `README.md` for the 1.0 implementation plan and open
|
||||||
and open design questions.
|
design questions.
|
||||||
|
|||||||
21
Dockerfile
21
Dockerfile
@@ -11,15 +11,32 @@ COPY . .
|
|||||||
# Touch .pb.go so make does not try to regenerate via protoc (file is committed)
|
# Touch .pb.go so make does not try to regenerate via protoc (file is committed)
|
||||||
RUN touch mfer/mf.pb.go
|
RUN touch mfer/mf.pb.go
|
||||||
|
|
||||||
RUN make fmt-check
|
# Go half of fmt-check only: this image has no node, so no prettier. The
|
||||||
|
# markdown half runs in the mdfmt stage below.
|
||||||
|
RUN make fmt-check-go
|
||||||
RUN make lint
|
RUN make lint
|
||||||
|
|
||||||
|
# Markdown/JSON format stage — prettier needs node, which the Go images
|
||||||
|
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
||||||
|
# 22.17.0 and yarn 1.22.22, the versions script/bootstrap pins.
|
||||||
|
FROM node@sha256:b04ce4ae4e95b522112c2e5c52f781471a5cbc3b594527bcddedee9bc48c03a0 AS mdfmt
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
COPY package.json yarn.lock ./
|
||||||
|
RUN yarn install --frozen-lockfile
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# No make in this image; call the script entrypoint directly.
|
||||||
|
RUN script/prettier --check
|
||||||
|
|
||||||
# Build stage — tests and compilation
|
# Build stage — tests and compilation
|
||||||
# golang:1.23 (2026-03-14)
|
# golang:1.23 (2026-03-14)
|
||||||
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
|
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
|
||||||
|
|
||||||
# Force BuildKit to run the lint stage by creating a stage dependency
|
# Force BuildKit to run the lint and mdfmt stages by creating stage dependencies
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
COPY --from=mdfmt /src/go.sum /dev/null
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
|
|||||||
33
FORMAT.md
33
FORMAT.md
@@ -5,9 +5,9 @@ Version 1.0
|
|||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
An `.mf` file is a binary manifest that describes a directory tree of files,
|
An `.mf` file is a binary manifest that describes a directory tree of files,
|
||||||
including their paths, sizes, and cryptographic checksums. It supports
|
including their paths, sizes, and cryptographic checksums. It supports optional
|
||||||
optional GPG signatures for integrity verification and optional timestamps
|
GPG signatures for integrity verification and optional timestamps for metadata
|
||||||
for metadata preservation.
|
preservation.
|
||||||
|
|
||||||
## File Structure
|
## File Structure
|
||||||
|
|
||||||
@@ -39,15 +39,15 @@ The outer message contains:
|
|||||||
|
|
||||||
### SHA-256 Hash
|
### SHA-256 Hash
|
||||||
|
|
||||||
The `sha256` field (104) covers the **compressed** `innerMessage` bytes.
|
The `sha256` field (104) covers the **compressed** `innerMessage` bytes. This
|
||||||
This allows verifying data integrity before decompression.
|
allows verifying data integrity before decompression.
|
||||||
|
|
||||||
## Compression
|
## Compression
|
||||||
|
|
||||||
The `innerMessage` field is compressed with [Zstandard (zstd)](https://facebook.github.io/zstd/).
|
The `innerMessage` field is compressed with
|
||||||
Implementations must enforce a decompression size limit to prevent
|
[Zstandard (zstd)](https://facebook.github.io/zstd/). Implementations must
|
||||||
decompression bombs. The reference implementation limits decompressed size to
|
enforce a decompression size limit to prevent decompression bombs. The reference
|
||||||
256 MB.
|
implementation limits decompressed size to 256 MB.
|
||||||
|
|
||||||
## Inner Message (`MFFile`)
|
## Inner Message (`MFFile`)
|
||||||
|
|
||||||
@@ -114,11 +114,11 @@ Where:
|
|||||||
|
|
||||||
- `ZNAVSRFG` is the magic bytes string (literal ASCII)
|
- `ZNAVSRFG` is the magic bytes string (literal ASCII)
|
||||||
- `<UUID>` is the hex-encoded UUID from the outer message
|
- `<UUID>` is the hex-encoded UUID from the outer message
|
||||||
- `<SHA256>` is the hex-encoded SHA-256 hash from the outer message (covering compressed data)
|
- `<SHA256>` is the hex-encoded SHA-256 hash from the outer message (covering
|
||||||
|
compressed data)
|
||||||
|
|
||||||
Components are separated by hyphens. The signature is produced by GPG over
|
Components are separated by hyphens. The signature is produced by GPG over this
|
||||||
this canonical string and stored in the `signature` field of the outer
|
canonical string and stored in the `signature` field of the outer message.
|
||||||
message.
|
|
||||||
|
|
||||||
## Deterministic Serialization
|
## Deterministic Serialization
|
||||||
|
|
||||||
@@ -134,10 +134,11 @@ changed).
|
|||||||
|
|
||||||
## MIME Type
|
## MIME Type
|
||||||
|
|
||||||
The recommended MIME type for `.mf` files is `application/octet-stream`.
|
The recommended MIME type for `.mf` files is `application/octet-stream`. The
|
||||||
The `.mf` file extension is the canonical identifier.
|
`.mf` file extension is the canonical identifier.
|
||||||
|
|
||||||
## Reference
|
## Reference
|
||||||
|
|
||||||
- Proto definition: [`mfer/mf.proto`](mfer/mf.proto)
|
- Proto definition: [`mfer/mf.proto`](mfer/mf.proto)
|
||||||
- Reference implementation: [git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
|
- Reference implementation:
|
||||||
|
[git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
|
||||||
|
|||||||
10
Makefile
10
Makefile
@@ -13,7 +13,7 @@ GOLDFLAGS += -X main.Version=$(VERSION)
|
|||||||
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
|
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
|
||||||
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
||||||
|
|
||||||
.PHONY: bootstrap setup docker default run ci test check lint fmt fmt-check hooks fixme
|
.PHONY: bootstrap setup docker default run ci test check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme
|
||||||
|
|
||||||
default: fmt test
|
default: fmt test
|
||||||
|
|
||||||
@@ -44,6 +44,14 @@ check:
|
|||||||
fmt-check:
|
fmt-check:
|
||||||
@script/fmt-check
|
@script/fmt-check
|
||||||
|
|
||||||
|
# Halves of fmt-check, for environments that have only one toolchain:
|
||||||
|
# the Docker lint stage has Go but no node, the markdown stage the reverse.
|
||||||
|
fmt-check-go:
|
||||||
|
@script/fmt-check-go
|
||||||
|
|
||||||
|
fmt-check-md:
|
||||||
|
@script/prettier --check
|
||||||
|
|
||||||
hooks:
|
hooks:
|
||||||
@script/install-precommit
|
@script/install-precommit
|
||||||
|
|
||||||
|
|||||||
419
README.md
419
README.md
@@ -1,76 +1,80 @@
|
|||||||
# mfer
|
# mfer
|
||||||
|
|
||||||
[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library
|
[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library and
|
||||||
and thin wrapper command-line utility written in [Go](https://golang.org)
|
thin wrapper command-line utility written in [Go](https://golang.org) and first
|
||||||
and first published in 2022 under the [WTFPL](https://wtfpl.net) (public
|
published in 2022 under the [WTFPL](https://wtfpl.net) (public domain) license.
|
||||||
domain) license. It specifies and generates `.mf` manifest files over a
|
It specifies and generates `.mf` manifest files over a directory tree of files
|
||||||
directory tree of files to encapsulate metadata about them (such as
|
to encapsulate metadata about them (such as cryptographic checksums or
|
||||||
cryptographic checksums or signatures over same) to aid in archiving,
|
signatures over same) to aid in archiving, downloading, and streaming, or
|
||||||
downloading, and streaming, or mirroring. The manifest files' data is
|
mirroring. The manifest files' data is serialized with Google's
|
||||||
serialized with Google's [protobuf serialization
|
[protobuf serialization format](https://developers.google.com/protocol-buffers).
|
||||||
format](https://developers.google.com/protocol-buffers). The structure of
|
The structure of these files can be found
|
||||||
these files can be found [in the format
|
[in the format specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
|
||||||
specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
|
which is included in the [project repository](https://git.eeqj.de/sneak/mfer).
|
||||||
which is included in the [project
|
|
||||||
repository](https://git.eeqj.de/sneak/mfer).
|
|
||||||
|
|
||||||
The current version is pre-1.0 and while the repo was published in 2022,
|
The current version is pre-1.0 and while the repo was published in 2022, there
|
||||||
there has not yet been any versioned release. [SemVer](https://semver.org)
|
has not yet been any versioned release. [SemVer](https://semver.org) will be
|
||||||
will be used for releases.
|
used for releases.
|
||||||
|
|
||||||
This project was started by [@sneak](https://sneak.berlin) to scratch an
|
This project was started by [@sneak](https://sneak.berlin) to scratch an itch in
|
||||||
itch in 2022 and is currently a one-person effort, though the goal is for
|
2022 and is currently a one-person effort, though the goal is for this to emerge
|
||||||
this to emerge as a de-facto standard and be incorporated into other
|
as a de-facto standard and be incorporated into other software. A compatible
|
||||||
software. A compatible javascript library is planned.
|
javascript library is planned.
|
||||||
|
|
||||||
# Build Status
|
# Build Status
|
||||||
|
|
||||||
CI runs via `script/cibuild` (`docker build .`), which executes `make
|
CI runs via `script/cibuild` (`docker build .`), which executes `make check`
|
||||||
check` (formatting, linting, tests). The `main` branch must always be
|
(formatting, linting, tests). The `main` branch must always be green.
|
||||||
green.
|
|
||||||
|
|
||||||
# Entrypoints
|
# Entrypoints
|
||||||
|
|
||||||
This repository adheres to the
|
This repository adheres to the
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||||
standard: normalized scripts in `script/` are the entrypoints for the
|
standard: normalized scripts in `script/` are the entrypoints for the
|
||||||
development workflow, and the Makefile targets are thin shims that call
|
development workflow, and the Makefile targets are thin shims that call them. We
|
||||||
them. We provide:
|
provide:
|
||||||
|
|
||||||
- `script/bootstrap` — install all dependencies (Go, golangci-lint, Go
|
- `script/bootstrap` — install all dependencies (Go, golangci-lint, Go module
|
||||||
module download), idempotently
|
download, and node/yarn plus the prettier version pinned in
|
||||||
|
`package.json`/`yarn.lock`), idempotently
|
||||||
- `script/setup` — make a fresh clone ready for development: runs
|
- `script/setup` — make a fresh clone ready for development: runs
|
||||||
`script/bootstrap`, then `script/install-precommit`
|
`script/bootstrap`, then `script/install-precommit`
|
||||||
- `script/projectname` — output the project name (`mfer`); used by other
|
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
||||||
scripts such as `script/docker`
|
such as `script/docker`
|
||||||
- `script/test` — run the test suite (`go test`), regenerating the
|
- `script/test` — run the test suite (`go test`), regenerating the protobuf code
|
||||||
protobuf code first if it is stale
|
first if it is stale
|
||||||
- `script/lint` — run `golangci-lint` and verify `gofmt` cleanliness
|
- `script/lint` — run `golangci-lint` and verify `gofmt` cleanliness
|
||||||
- `script/fmt` — format all code and docs (writes): `gofumpt`,
|
- `script/fmt` — format all code and docs (writes): `gofumpt`,
|
||||||
`golangci-lint run --fix`, and prettier for JSON/Markdown
|
`golangci-lint run --fix`, and `script/prettier --write`
|
||||||
- `script/fmt-check` — check formatting without writing
|
- `script/prettier` — run prettier over the repository's canonical file set
|
||||||
- `script/check` — run `script/test`, `script/lint`, and
|
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
||||||
`script/fmt-check`
|
`--check`; the single definition of that file set, so `script/fmt` and
|
||||||
|
`script/fmt-check` cannot disagree about it
|
||||||
|
- `script/fmt-check` — check formatting without writing: `script/fmt-check-go`
|
||||||
|
plus `script/prettier --check`
|
||||||
|
- `script/fmt-check-go` — the Go half of `script/fmt-check`, on its own, for the
|
||||||
|
Docker lint stage, whose image has no node
|
||||||
|
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
||||||
- `script/docker` — build the Docker image tagged with the project name
|
- `script/docker` — build the Docker image tagged with the project name
|
||||||
- `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile
|
- `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile runs the
|
||||||
runs the checks)
|
checks)
|
||||||
- `script/precommit` — pre-commit checks: `go mod tidy` verification,
|
- `script/precommit` — pre-commit checks: `go mod tidy` verification, then
|
||||||
then `script/check`
|
`script/check`
|
||||||
- `script/install-precommit` — install the git pre-commit hook that
|
- `script/install-precommit` — install the git pre-commit hook that runs
|
||||||
runs `script/precommit`
|
`script/precommit`
|
||||||
|
|
||||||
# Participation
|
# Participation
|
||||||
|
|
||||||
The community is as yet nonexistent so there are no defined policies or
|
The community is as yet nonexistent so there are no defined policies or norms
|
||||||
norms yet. Primary development happens on a privately-run Gitea instance at
|
yet. Primary development happens on a privately-run Gitea instance at
|
||||||
[https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) and issues
|
[https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) and issues are
|
||||||
are [tracked there](https://git.eeqj.de/sneak/mfer/issues).
|
[tracked there](https://git.eeqj.de/sneak/mfer/issues).
|
||||||
|
|
||||||
Changes must always be formatted with a standard `go fmt`, syntactically
|
Changes must always be formatted with a standard `go fmt`, syntactically valid,
|
||||||
valid, and must pass the linting defined in the repository (presently only
|
and must pass the linting defined in the repository (presently only the
|
||||||
the `golangci-lint` defaults), which can be run with a `make lint`. The
|
`golangci-lint` defaults), which can be run with a `make lint`. The `main`
|
||||||
`main` branch is protected and all changes must be made via [pull
|
branch is protected and all changes must be made via
|
||||||
requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
|
[pull requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
|
||||||
Any changes submitted to this project must also be
|
Any changes submitted to this project must also be
|
||||||
[WTFPL-licensed](https://wtfpl.net) to be considered.
|
[WTFPL-licensed](https://wtfpl.net) to be considered.
|
||||||
|
|
||||||
@@ -81,14 +85,14 @@ tooling requirements, and workflow conventions.
|
|||||||
|
|
||||||
Given a plain URL, there is no standard way to safely and programmatically
|
Given a plain URL, there is no standard way to safely and programmatically
|
||||||
download everything "under" that URL path. `wget -r` can traverse directory
|
download everything "under" that URL path. `wget -r` can traverse directory
|
||||||
listings if they're enabled, but every server has a different format, and
|
listings if they're enabled, but every server has a different format, and this
|
||||||
this does not verify cryptographic integrity of the files, or enable them to
|
does not verify cryptographic integrity of the files, or enable them to be
|
||||||
be fetched using a different protocol other than HTTP/s.
|
fetched using a different protocol other than HTTP/s.
|
||||||
|
|
||||||
Currently, the solution that people are using are sidecar files in the
|
Currently, the solution that people are using are sidecar files in the format of
|
||||||
format of `SHASUMS` checksum files, as well as a `SHASUMS.asc` PGP detached
|
`SHASUMS` checksum files, as well as a `SHASUMS.asc` PGP detached signature.
|
||||||
signature. This is not checksum-algorithm-agnostic and the sidecar file is
|
This is not checksum-algorithm-agnostic and the sidecar file is not always
|
||||||
not always consistently named.
|
consistently named.
|
||||||
|
|
||||||
Real issues I face:
|
Real issues I face:
|
||||||
|
|
||||||
@@ -98,35 +102,34 @@ Real issues I face:
|
|||||||
- when I want to mirror an HTTP archive, I have to use special tools like
|
- when I want to mirror an HTTP archive, I have to use special tools like
|
||||||
debmirror that understand the archive format
|
debmirror that understand the archive format
|
||||||
- the debian repository metadata structure is hot garbage
|
- the debian repository metadata structure is hot garbage
|
||||||
- when I download a large file via HTTP, I have no way of knowing if the
|
- when I download a large file via HTTP, I have no way of knowing if the file
|
||||||
file content is what it's supposed to be
|
content is what it's supposed to be
|
||||||
|
|
||||||
# Proposed Solution
|
# Proposed Solution
|
||||||
|
|
||||||
A standard, a manifest file format, and a tool for generating same.
|
A standard, a manifest file format, and a tool for generating same.
|
||||||
|
|
||||||
The manifest file would be called `index.mf`, and the tool for generating such would be called `mfer`.
|
The manifest file would be called `index.mf`, and the tool for generating such
|
||||||
|
would be called `mfer`.
|
||||||
|
|
||||||
The manifest file would do several important things:
|
The manifest file would do several important things:
|
||||||
|
|
||||||
- have a standard filename, so if given
|
- have a standard filename, so if given `https://example.com/downloadpackage/`
|
||||||
`https://example.com/downloadpackage/` one could fetch
|
one could fetch `https://example.com/downloadpackage/index.mf` to enumerate
|
||||||
`https://example.com/downloadpackage/index.mf` to enumerate the full
|
the full directory listing.
|
||||||
directory listing.
|
|
||||||
- contain a version field for extensibility
|
- contain a version field for extensibility
|
||||||
- contain structured data (protobuf, json, or cbor)
|
- contain structured data (protobuf, json, or cbor)
|
||||||
- provide an inner signed container, so that the manifest file itself can
|
- provide an inner signed container, so that the manifest file itself can embed
|
||||||
embed a signature and a public key alongside in a single file
|
a signature and a public key alongside in a single file
|
||||||
- contain a list of files, each with a relative path to the manifest
|
- contain a list of files, each with a relative path to the manifest
|
||||||
- contain manifest timestamp
|
- contain manifest timestamp
|
||||||
- contain ctime/mtime information for files so that file metadata can be
|
- contain ctime/mtime information for files so that file metadata can be
|
||||||
preserved
|
preserved
|
||||||
- contain cryptographic checksums in several different algorithms for each
|
- contain cryptographic checksums in several different algorithms for each file
|
||||||
file
|
|
||||||
- probably encoded with multihash to indicate algo + hash
|
- probably encoded with multihash to indicate algo + hash
|
||||||
- sha256 at the minimum
|
- sha256 at the minimum
|
||||||
- would be nice to include an IPFS/IPLD CIDv1 root hash for each file,
|
- would be nice to include an IPFS/IPLD CIDv1 root hash for each file, which
|
||||||
which likely involves doing an ipfs file object chunking
|
likely involves doing an ipfs file object chunking
|
||||||
- maybe even including the complete IPFS/IPLD directory tree objects and
|
- maybe even including the complete IPFS/IPLD directory tree objects and
|
||||||
chunklists?
|
chunklists?
|
||||||
- this is because generating an `index.mf` does not imply publishing on
|
- this is because generating an `index.mf` does not imply publishing on
|
||||||
@@ -146,9 +149,8 @@ The manifest file would do several important things:
|
|||||||
# Non-Goals
|
# Non-Goals
|
||||||
|
|
||||||
- Manifest generation speed
|
- Manifest generation speed
|
||||||
- likely involves IPFS chunking, bittorrent chunking, and several
|
- likely involves IPFS chunking, bittorrent chunking, and several different
|
||||||
different cryptographic hash functions over the entirety of each and
|
cryptographic hash functions over the entirety of each and every file
|
||||||
every file
|
|
||||||
- Small manifest file size (within reason)
|
- Small manifest file size (within reason)
|
||||||
- 30MiB files are "small" these days, given modern storage/bandwidth
|
- 30MiB files are "small" these days, given modern storage/bandwidth
|
||||||
- metadata size should not be used as an excuse to sacrifice utility (such
|
- metadata size should not be used as an excuse to sacrifice utility (such
|
||||||
@@ -156,13 +158,14 @@ The manifest file would do several important things:
|
|||||||
|
|
||||||
# Open Questions
|
# Open Questions
|
||||||
|
|
||||||
- Should the manifest file include checksums of individual file chunks, or just for the whole assembled file?
|
- Should the manifest file include checksums of individual file chunks, or just
|
||||||
|
for the whole assembled file?
|
||||||
|
|
||||||
- If so, should the chunksize be fixed or dynamic?
|
- If so, should the chunksize be fixed or dynamic?
|
||||||
|
|
||||||
- Should the manifest signature format be GnuPG signatures, or those from
|
- Should the manifest signature format be GnuPG signatures, or those from
|
||||||
OpenBSD's signify (of which there is a good [golang
|
OpenBSD's signify (of which there is a good
|
||||||
implementation](https://github.com/frankbraun/gosignify)?
|
[golang implementation](https://github.com/frankbraun/gosignify)?
|
||||||
|
|
||||||
- Should the on-disk serialization format be proto3 or json?
|
- Should the on-disk serialization format be proto3 or json?
|
||||||
|
|
||||||
@@ -171,8 +174,8 @@ The manifest file would do several important things:
|
|||||||
- `mfer gen` / `mfer gen .`
|
- `mfer gen` / `mfer gen .`
|
||||||
- recurses under current directory and writes out an `index.mf`
|
- recurses under current directory and writes out an `index.mf`
|
||||||
- `mfer check` / `mfer check .`
|
- `mfer check` / `mfer check .`
|
||||||
- verifies checksums of all files in manifest, displaying error and
|
- verifies checksums of all files in manifest, displaying error and exiting
|
||||||
exiting nonzero if any files are missing or corrupted
|
nonzero if any files are missing or corrupted
|
||||||
- `mfer fetch https://example.com/stuff/`
|
- `mfer fetch https://example.com/stuff/`
|
||||||
- fetches `/stuff/index.mf` and downloads all files listed in manifest,
|
- fetches `/stuff/index.mf` and downloads all files listed in manifest,
|
||||||
optionally resuming any that already exist locally, and assures
|
optionally resuming any that already exist locally, and assures
|
||||||
@@ -196,27 +199,28 @@ The manifest file would do several important things:
|
|||||||
- (fetches `https://example.com/manifestdirectory/index.mf`, downloads and
|
- (fetches `https://example.com/manifestdirectory/index.mf`, downloads and
|
||||||
checksums all files, resumes any that exist locally already)
|
checksums all files, resumes any that exist locally already)
|
||||||
- `mfer fetch https://example.com/manifestdirectory/`
|
- `mfer fetch https://example.com/manifestdirectory/`
|
||||||
- a command line option to zero/omit mtime/ctime, as well as manifest
|
- a command line option to zero/omit mtime/ctime, as well as manifest timestamp,
|
||||||
timestamp, and sort all directory listings so that manifest file
|
and sort all directory listings so that manifest file generation is
|
||||||
generation is deterministic/reproducible
|
deterministic/reproducible
|
||||||
- URL format `mfer fetch https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2`
|
- URL format
|
||||||
to assert in the URL which PGP signing key should be used in the manifest,
|
`mfer fetch https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2`
|
||||||
so that shared URLs have a cryptographic trust root
|
to assert in the URL which PGP signing key should be used in the manifest, so
|
||||||
- a "well-known" key in the manifest that maps well known keys (could reuse
|
that shared URLs have a cryptographic trust root
|
||||||
the http spec) to specific file paths in the manifest.
|
- a "well-known" key in the manifest that maps well known keys (could reuse the
|
||||||
- example: a `berlin.sneak.app.slideshow` key that maps to a json
|
http spec) to specific file paths in the manifest.
|
||||||
slideshow config listing what image paths to show, and for how long, and
|
- example: a `berlin.sneak.app.slideshow` key that maps to a json slideshow
|
||||||
in what order
|
config listing what image paths to show, and for how long, and in what
|
||||||
|
order
|
||||||
|
|
||||||
# Use Cases
|
# Use Cases
|
||||||
|
|
||||||
## Web Images
|
## Web Images
|
||||||
|
|
||||||
I'd like to be able to put a bunch of images into a directory, generate a
|
I'd like to be able to put a bunch of images into a directory, generate a
|
||||||
manifest, and then point a slideshow client (such as an ambient display, or
|
manifest, and then point a slideshow client (such as an ambient display, or a
|
||||||
a react app with the target directory in a query string arg) at that
|
react app with the target directory in a query string arg) at that statically
|
||||||
statically hosted directory, and have it discover the full list of images
|
hosted directory, and have it discover the full list of images available at that
|
||||||
available at that URL.
|
URL.
|
||||||
|
|
||||||
## Software Distribution
|
## Software Distribution
|
||||||
|
|
||||||
@@ -226,20 +230,20 @@ resumably by either HTTP or IPFS/BitTorrent without a .torrent file.
|
|||||||
## Filesystem Archive Integrity
|
## Filesystem Archive Integrity
|
||||||
|
|
||||||
I use filesystems that don't include data checksums, and I would like a
|
I use filesystems that don't include data checksums, and I would like a
|
||||||
cryptographically signed checksum file so that I can later verify that a set
|
cryptographically signed checksum file so that I can later verify that a set of
|
||||||
of archive files have not been modified, none are missing, and that the
|
archive files have not been modified, none are missing, and that the checksums
|
||||||
checksums have not been altered in storage by a second party.
|
have not been altered in storage by a second party.
|
||||||
|
|
||||||
## Filesystem-Independent Checksums
|
## Filesystem-Independent Checksums
|
||||||
|
|
||||||
I would like to be able to plug in a hard drive or flash drive and, if there
|
I would like to be able to plug in a hard drive or flash drive and, if there is
|
||||||
is an `index.mf` in the root, automatically detect missing/corrupted files,
|
an `index.mf` in the root, automatically detect missing/corrupted files,
|
||||||
regardless of filesystem format.
|
regardless of filesystem format.
|
||||||
|
|
||||||
# Collaboration
|
# Collaboration
|
||||||
|
|
||||||
Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your
|
Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your desired
|
||||||
desired username for an account on this Gitea instance.
|
username for an account on this Gitea instance.
|
||||||
|
|
||||||
# TODO: Remaining Work for 1.0
|
# TODO: Remaining Work for 1.0
|
||||||
|
|
||||||
@@ -250,114 +254,108 @@ inline below each question.
|
|||||||
|
|
||||||
### Format Design
|
### Format Design
|
||||||
|
|
||||||
**1. Should `MFFileChecksum` be simplified?** Currently it's a separate
|
**1. Should `MFFileChecksum` be simplified?** Currently it's a separate message
|
||||||
message wrapping a single `bytes multiHash` field. Since multihash
|
wrapping a single `bytes multiHash` field. Since multihash already
|
||||||
already self-describes the algorithm, `repeated bytes hashes` directly on
|
self-describes the algorithm, `repeated bytes hashes` directly on `MFFilePath`
|
||||||
`MFFilePath` would be simpler and reduce per-file protobuf overhead. Is
|
would be simpler and reduce per-file protobuf overhead. Is the extra message
|
||||||
the extra message layer intentional (e.g. planning to add per-hash
|
layer intentional (e.g. planning to add per-hash metadata like `verified_at`)?
|
||||||
metadata like `verified_at`)?
|
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**2. Should file permissions/mode be stored?** The format stores
|
**2. Should file permissions/mode be stored?** The format stores mtime/ctime but
|
||||||
mtime/ctime but not Unix file permissions. For archival use this may not
|
not Unix file permissions. For archival use this may not matter, but for
|
||||||
matter, but for software distribution or filesystem restoration it's a
|
software distribution or filesystem restoration it's a gap. Should we reserve a
|
||||||
gap. Should we reserve a field now (e.g. `optional uint32 mode = 305`)
|
field now (e.g. `optional uint32 mode = 305`) even if we don't populate it yet?
|
||||||
even if we don't populate it yet?
|
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**3. Should `atime` be removed from the schema?** Access time is
|
**3. Should `atime` be removed from the schema?** Access time is volatile,
|
||||||
volatile, non-deterministic, and often disabled (`noatime`). Including it
|
non-deterministic, and often disabled (`noatime`). Including it means two
|
||||||
means two manifests of the same directory at different times will differ,
|
manifests of the same directory at different times will differ, which conflicts
|
||||||
which conflicts with the determinism goal. Remove it, or document it as
|
with the determinism goal. Remove it, or document it as "never set by default"?
|
||||||
"never set by default"?
|
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**4. What are the path normalization rules?** The proto has `string path`
|
**4. What are the path normalization rules?** The proto has `string path` with
|
||||||
with no specification about: always forward-slash? Must be relative? No
|
no specification about: always forward-slash? Must be relative? No `..`
|
||||||
`..` components allowed? UTF-8 NFC vs NFD normalization (macOS vs
|
components allowed? UTF-8 NFC vs NFD normalization (macOS vs Linux)? Max path
|
||||||
Linux)? Max path length? This is a security issue (path traversal) and a
|
length? This is a security issue (path traversal) and a cross-platform
|
||||||
cross-platform compatibility issue. What rules should the spec mandate?
|
compatibility issue. What rules should the spec mandate?
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**5. Should we add a version byte after the magic?** Currently
|
**5. Should we add a version byte after the magic?** Currently `ZNAVSRFG` is
|
||||||
`ZNAVSRFG` is followed immediately by protobuf. Adding a version byte
|
followed immediately by protobuf. Adding a version byte (`ZNAVSRFG\x01`) would
|
||||||
(`ZNAVSRFG\x01`) would allow future framing changes without requiring
|
allow future framing changes without requiring protobuf parsing to detect the
|
||||||
protobuf parsing to detect the version. `MFFileOuter.Version` serves
|
version. `MFFileOuter.Version` serves this purpose but requires successful
|
||||||
this purpose but requires successful deserialization to read. Worth the
|
deserialization to read. Worth the extra byte?
|
||||||
extra byte?
|
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**6. Should we add a length-prefix after the magic?** Protobuf is not
|
**6. Should we add a length-prefix after the magic?** Protobuf is not
|
||||||
self-delimiting. If we ever want to concatenate manifests or append data
|
self-delimiting. If we ever want to concatenate manifests or append data after
|
||||||
after the protobuf, the current framing is insufficient. Add a varint or
|
the protobuf, the current framing is insufficient. Add a varint or fixed-width
|
||||||
fixed-width length-prefix?
|
length-prefix?
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
### Signature Design
|
### Signature Design
|
||||||
|
|
||||||
**7. What does the outer SHA-256 hash cover — compressed or uncompressed
|
**7. What does the outer SHA-256 hash cover — compressed or uncompressed data?**
|
||||||
data?** The code currently hashes compressed data (good for verifying
|
The code currently hashes compressed data (good for verifying before
|
||||||
before decompression), but this should be explicitly documented. Which is
|
decompression), but this should be explicitly documented. Which is the intended
|
||||||
the intended behavior?
|
behavior?
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**8. Should `signatureString()` sign raw bytes instead of a hex-encoded
|
**8. Should `signatureString()` sign raw bytes instead of a hex-encoded
|
||||||
string?** Currently the canonical string is `MAGIC-UUID-MULTIHASH` with
|
string?** Currently the canonical string is `MAGIC-UUID-MULTIHASH` with hex
|
||||||
hex encoding, which adds a transformation layer. Signing the raw `sha256`
|
encoding, which adds a transformation layer. Signing the raw `sha256` bytes (or
|
||||||
bytes (or compressed `innerMessage` directly) would be simpler. Keep the
|
compressed `innerMessage` directly) would be simpler. Keep the string format or
|
||||||
string format or switch to raw bytes?
|
switch to raw bytes?
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**9. Should we support detached signature files (`.mf.sig`)?** Embedded
|
**9. Should we support detached signature files (`.mf.sig`)?** Embedded
|
||||||
signatures are better for single-file distribution. Detached `.mf.sig`
|
signatures are better for single-file distribution. Detached `.mf.sig` files
|
||||||
files follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are
|
follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are simpler for HTTP
|
||||||
simpler for HTTP serving. Support both modes?
|
serving. Support both modes?
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**10. GPG vs pure-Go crypto for signatures?** Shelling out to `gpg` is
|
**10. GPG vs pure-Go crypto for signatures?** Shelling out to `gpg` is fragile
|
||||||
fragile (may not be installed, version-dependent output).
|
(may not be installed, version-dependent output).
|
||||||
`github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could
|
`github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could use
|
||||||
use Ed25519/signify (simpler, no key management). Which direction?
|
Ed25519/signify (simpler, no key management). Which direction?
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
### Implementation Design
|
### Implementation Design
|
||||||
|
|
||||||
**11. Should manifests be deterministic by default?** This means: sort
|
**11. Should manifests be deterministic by default?** This means: sort file
|
||||||
file entries by path, omit `createdAt` timestamp (or make it opt-in), no
|
entries by path, omit `createdAt` timestamp (or make it opt-in), no `atime`.
|
||||||
`atime`. Should determinism be the default, with a
|
Should determinism be the default, with a `--include-timestamps` flag to opt in?
|
||||||
`--include-timestamps` flag to opt in?
|
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**12. Should we consolidate or keep both scanner/checker
|
**12. Should we consolidate or keep both scanner/checker implementations?**
|
||||||
implementations?** There are two parallel implementations:
|
There are two parallel implementations: `mfer/scanner.go` + `mfer/checker.go`
|
||||||
`mfer/scanner.go` + `mfer/checker.go` (typed with `FileSize`,
|
(typed with `FileSize`, `RelFilePath`) and `internal/scanner/` +
|
||||||
`RelFilePath`) and `internal/scanner/` + `internal/checker/` (raw
|
`internal/checker/` (raw `int64`, `string`). The `mfer/` versions are superior.
|
||||||
`int64`, `string`). The `mfer/` versions are superior. Delete the
|
Delete the `internal/` versions?
|
||||||
`internal/` versions?
|
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**13. Should the `manifest` type be exported?** Currently unexported with
|
**13. Should the `manifest` type be exported?** Currently unexported with
|
||||||
exported constructors (`NewManifestFromReader`, `NewManifestFromFile`).
|
exported constructors (`NewManifestFromReader`, `NewManifestFromFile`).
|
||||||
Consumers can't declare `var m *mfer.manifest`. Export the type, or
|
Consumers can't declare `var m *mfer.manifest`. Export the type, or define an
|
||||||
define an interface?
|
interface?
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
**14. What should the Go module path be for 1.0?** Currently
|
**14. What should the Go module path be for 1.0?** Currently
|
||||||
`sneak.berlin/go/mfer` in `go.mod` but `git.eeqj.de/sneak/mfer/mfer` in
|
`sneak.berlin/go/mfer` in `go.mod` but `git.eeqj.de/sneak/mfer/mfer` in the
|
||||||
the proto `go_package` option. Which is canonical?
|
proto `go_package` option. Which is canonical?
|
||||||
|
|
||||||
> _answer:_
|
> _answer:_
|
||||||
|
|
||||||
@@ -375,39 +373,37 @@ the proto `go_package` option. Which is canonical?
|
|||||||
- [ ] Resolve proto `go_package` path inconsistency
|
- [ ] Resolve proto `go_package` path inconsistency
|
||||||
(`git.eeqj.de/sneak/mfer/mfer` vs `sneak.berlin/go/mfer`)
|
(`git.eeqj.de/sneak/mfer/mfer` vs `sneak.berlin/go/mfer`)
|
||||||
- [ ] Specify path invariants — add proto comments requiring UTF-8,
|
- [ ] Specify path invariants — add proto comments requiring UTF-8,
|
||||||
forward-slash, relative paths, no `..`, no leading `/`; validate
|
forward-slash, relative paths, no `..`, no leading `/`; validate in
|
||||||
in `Builder.AddFile` and `Builder.AddFileWithHash` (pending design
|
`Builder.AddFile` and `Builder.AddFileWithHash` (pending design question
|
||||||
question answer)
|
|
||||||
- [ ] Remove or deprecate `atime` from proto (pending design question
|
|
||||||
answer)
|
answer)
|
||||||
- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future
|
- [ ] Remove or deprecate `atime` from proto (pending design question answer)
|
||||||
file permissions (pending design question answer)
|
- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future file
|
||||||
- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version
|
permissions (pending design question answer)
|
||||||
1 (pending design question answer)
|
- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version 1
|
||||||
- [ ] Write format specification document — separate from README:
|
(pending design question answer)
|
||||||
magic, outer structure, compression, inner structure, path
|
- [ ] Write format specification document — separate from README: magic, outer
|
||||||
invariants, signature scheme, canonical serialization
|
structure, compression, inner structure, path invariants, signature
|
||||||
|
scheme, canonical serialization
|
||||||
|
|
||||||
### Library
|
### Library
|
||||||
|
|
||||||
- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate
|
- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate on
|
||||||
on `mfer/` package versions; update CLI code (pending design
|
`mfer/` package versions; update CLI code (pending design question answer)
|
||||||
question answer)
|
- [ ] Add deterministic file ordering — sort entries by path (lexicographic,
|
||||||
- [ ] Add deterministic file ordering — sort entries by path
|
byte-order) in `Builder.Build()`; add test asserting byte-identical output
|
||||||
(lexicographic, byte-order) in `Builder.Build()`; add test
|
from two runs
|
||||||
asserting byte-identical output from two runs
|
- [ ] Add decompression size limit — `io.LimitReader` in `deserializeInner()`
|
||||||
- [ ] Add decompression size limit — `io.LimitReader` in
|
with `m.pbOuter.Size` as bound
|
||||||
`deserializeInner()` with `m.pbOuter.Size` as bound
|
- [ ] Fix `errors.Is` dead code in checker — replace with `os.IsNotExist(err)`
|
||||||
- [ ] Fix `errors.Is` dead code in checker — replace with
|
or `errors.Is(err, fs.ErrNotExist)`
|
||||||
`os.IsNotExist(err)` or `errors.Is(err, fs.ErrNotExist)`
|
- [ ] Fix `AddFile` to verify size — check `totalRead == size` after reading,
|
||||||
- [ ] Fix `AddFile` to verify size — check `totalRead == size` after
|
return error on mismatch
|
||||||
reading, return error on mismatch
|
- [ ] Export the `manifest` type or define a public interface (pending design
|
||||||
- [ ] Export the `manifest` type or define a public interface (pending
|
question answer) — currently consumers cannot hold a reference to a loaded
|
||||||
design question answer) — currently consumers cannot hold a reference
|
manifest in their own type declarations
|
||||||
to a loaded manifest in their own type declarations
|
- [ ] Replace GPG subprocess calls with pure-Go crypto (pending design question
|
||||||
- [ ] Replace GPG subprocess calls with pure-Go crypto (pending design
|
answer) — current implementation shells out to `gpg` which may not be
|
||||||
question answer) — current implementation shells out to `gpg` which
|
installed
|
||||||
may not be installed
|
|
||||||
- [ ] Add timeout to any remaining subprocess calls
|
- [ ] Add timeout to any remaining subprocess calls
|
||||||
|
|
||||||
### CLI
|
### CLI
|
||||||
@@ -416,35 +412,33 @@ the proto `go_package` option. Which is canonical?
|
|||||||
(`--include-dotfiles`, `--follow-symlinks`)
|
(`--include-dotfiles`, `--follow-symlinks`)
|
||||||
- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or
|
- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or
|
||||||
`url.JoinPath()` instead of string concatenation
|
`url.JoinPath()` instead of string concatenation
|
||||||
- [ ] Add progress rate-limiting to Checker — throttle to once per
|
- [ ] Add progress rate-limiting to Checker — throttle to once per second,
|
||||||
second, matching Scanner
|
matching Scanner
|
||||||
- [ ] Add `--deterministic` flag or make it default — omit `createdAt`,
|
- [ ] Add `--deterministic` flag or make it default — omit `createdAt`, sort
|
||||||
sort files (pending design question answer)
|
files (pending design question answer)
|
||||||
- [ ] Wire `--version` flag properly (currently only a `version`
|
- [ ] Wire `--version` flag properly (currently only a `version` subcommand
|
||||||
subcommand exists; top-level `--version` shows urfave/cli generic
|
exists; top-level `--version` shows urfave/cli generic output)
|
||||||
output)
|
- [ ] Add retry logic to `fetch` — currently no retries on transient HTTP
|
||||||
- [ ] Add retry logic to `fetch` — currently no retries on transient
|
errors; needs exponential backoff
|
||||||
HTTP errors; needs exponential backoff
|
- [ ] `fetch` command uses bare `http.Get` with no timeout — needs `http.Client`
|
||||||
- [ ] `fetch` command uses bare `http.Get` with no timeout — needs
|
with configurable timeout
|
||||||
`http.Client` with configurable timeout
|
|
||||||
|
|
||||||
### Testing & Robustness
|
### Testing & Robustness
|
||||||
|
|
||||||
- [ ] Add fuzzing tests for `NewManifestFromReader` — protobuf
|
- [ ] Add fuzzing tests for `NewManifestFromReader` — protobuf deserialization
|
||||||
deserialization of untrusted input needs fuzz coverage
|
of untrusted input needs fuzz coverage
|
||||||
- [ ] Add integration test for `freshen` CLI command — current tests
|
- [ ] Add integration test for `freshen` CLI command — current tests only verify
|
||||||
only verify setup, not the actual freshen operation end-to-end
|
setup, not the actual freshen operation end-to-end
|
||||||
- [ ] Add test for `fetch` CLI command end-to-end (currently only
|
- [ ] Add test for `fetch` CLI command end-to-end (currently only `downloadFile`
|
||||||
`downloadFile` is tested)
|
is tested)
|
||||||
|
|
||||||
### Documentation
|
### Documentation
|
||||||
|
|
||||||
- [ ] Promote `FORMAT.md` as primary spec reference; README should link
|
- [ ] Promote `FORMAT.md` as primary spec reference; README should link to it
|
||||||
to it more prominently
|
more prominently
|
||||||
- [ ] Audit and update all error messages for consistency and
|
- [ ] Audit and update all error messages for consistency and helpfulness
|
||||||
helpfulness
|
- [ ] Document the signature scheme more thoroughly (canonical string format,
|
||||||
- [ ] Document the signature scheme more thoroughly (canonical string
|
verification steps)
|
||||||
format, verification steps)
|
|
||||||
|
|
||||||
### Release
|
### Release
|
||||||
|
|
||||||
@@ -465,7 +459,8 @@ the proto `go_package` option. Which is canonical?
|
|||||||
## Links
|
## Links
|
||||||
|
|
||||||
- Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
|
- Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
|
||||||
- Issues: [https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
|
- Issues:
|
||||||
|
[https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
|
||||||
|
|
||||||
# Authors
|
# Authors
|
||||||
|
|
||||||
|
|||||||
124
TODO.md
124
TODO.md
@@ -10,102 +10,100 @@
|
|||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
pre-1.0. No git tags. README section "TODO: Remaining Work for 1.0" lists
|
pre-1.0. No git tags. README section "TODO: Remaining Work for 1.0" lists open
|
||||||
open design questions and implementation tasks; policy compliance work is
|
design questions and implementation tasks; policy compliance work is in flight
|
||||||
in flight and unmerged.
|
and unmerged.
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
Work through the remaining compliance items folded from the 2026-07-02
|
Work through the remaining compliance items folded from the 2026-07-02 audit
|
||||||
audit (the first group under Future Steps): `.editorconfig`, `.gitignore`
|
(the first group under Future Steps): `.editorconfig`, `.gitignore` coverage,
|
||||||
coverage, gofumpt-based `fmt-check`, README "Getting Started", and the
|
gofumpt-based `fmt-check`, README "Getting Started", and the rest.
|
||||||
rest. `.golangci.yml` and `TODO.md` are tracked and committed as of
|
`.golangci.yml` and `TODO.md` are tracked and committed as of 2026-08-07, so the
|
||||||
2026-08-07, so the only thing left of the `chore/align-repo-policies`
|
only thing left of the `chore/align-repo-policies` branch is the list below.
|
||||||
branch is the list below.
|
|
||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-08-09: added `.prettierrc`/`.prettierignore`, gave `script/fmt` and
|
||||||
|
`script/fmt-check` one shared prettier file set via `script/prettier`, dropped
|
||||||
|
the `|| true` that hid prettier failures, and added a node-based Dockerfile
|
||||||
|
stage so a markdown formatting violation fails `docker build .` (#69)
|
||||||
- 2026-08-07: updated golangci-lint to v2.12.2 everywhere it is pinned
|
- 2026-08-07: updated golangci-lint to v2.12.2 everywhere it is pinned
|
||||||
(`Makefile`, `Dockerfile`), added the canonical `.golangci.yml`
|
(`Makefile`, `Dockerfile`), added the canonical `.golangci.yml`
|
||||||
(`default: all`), and fixed all resulting lint findings across the
|
(`default: all`), and fixed all resulting lint findings across the codebase
|
||||||
codebase
|
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
|
||||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
shims, README Entrypoints section
|
||||||
Makefile shims, README Entrypoints section
|
- 2026-07-03: aligned repo tooling, docs, and config with standardized policies
|
||||||
- 2026-07-03: aligned repo tooling, docs, and config with standardized
|
(7d9a138, on chore/align-repo-policies, unmerged)
|
||||||
policies (7d9a138, on chore/align-repo-policies, unmerged)
|
|
||||||
- 2026-06-28: moved to standardized repo policies (#56, on main)
|
- 2026-06-28: moved to standardized repo policies (#56, on main)
|
||||||
- 2026-04-07: added 1.0 roadmap as README TODO section, removed old
|
- 2026-04-07: added 1.0 roadmap as README TODO section, removed old TODO.md
|
||||||
TODO.md (#54)
|
(#54)
|
||||||
- 2026-03-20: added Gitea Actions CI workflow (#53)
|
- 2026-03-20: added Gitea Actions CI workflow (#53)
|
||||||
- 2026-03-17: added REPO_POLICIES.md, renamed CLAUDE.md to AGENTS.md (#51);
|
- 2026-03-17: added REPO_POLICIES.md, renamed CLAUDE.md to AGENTS.md (#51);
|
||||||
removed committed .index.mf (#52)
|
removed committed .index.mf (#52)
|
||||||
- 2026-03-15: split Dockerfile with pre-built golangci-lint stage for
|
- 2026-03-15: split Dockerfile with pre-built golangci-lint stage for faster CI
|
||||||
faster CI (#45)
|
(#45)
|
||||||
- 2026-03-01: 1.0 quality polish: code review, tests, bug fixes, docs (#32)
|
- 2026-03-01: 1.0 quality polish: code review, tests, bug fixes, docs (#32)
|
||||||
- 2026-02-20: deterministic file ordering in Builder.Build() (#28);
|
- 2026-02-20: deterministic file ordering in Builder.Build() (#28); removed
|
||||||
removed committed vendor/modcache archives (#35)
|
committed vendor/modcache archives (#35)
|
||||||
- 2026-02-08: added --seed flag for deterministic manifest UUID
|
- 2026-02-08: added --seed flag for deterministic manifest UUID
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Compliance (fold of TODO.md audit 2026-07-02; verify which items the
|
- Compliance (fold of TODO.md audit 2026-07-02; verify which items the in-flight
|
||||||
in-flight branch already closes, then check off):
|
branch already closes, then check off):
|
||||||
- Add .editorconfig (canonical copy from sneak/prompts)
|
- Add .editorconfig (canonical copy from sneak/prompts)
|
||||||
- Make .gitignore cover secrets (.env, _.key, _.pem), OS files
|
- Make .gitignore cover secrets (.env, _.key, _.pem), OS files (.DS_Store),
|
||||||
(.DS_Store), and editor files (_.swp, _~)
|
and editor files (_.swp, _~)
|
||||||
- Make fmt-check/lint verify with gofumpt, not gofmt -l, so
|
- Make fmt-check/lint verify with gofumpt, not gofmt -l, so `make check`
|
||||||
`make check` matches what `make fmt` writes
|
matches what `make fmt` writes
|
||||||
- Add README "Getting Started" section with copy-pasteable
|
- Add README "Getting Started" section with copy-pasteable install/usage
|
||||||
install/usage block
|
block
|
||||||
- Move FORMAT.md from repo root to docs/ and update the AGENTS.md
|
- Move FORMAT.md from repo root to docs/ and update the AGENTS.md reference
|
||||||
reference
|
|
||||||
- Pin Makefile-installed Go tools (`protoc-gen-go@v1.28.1`,
|
- Pin Makefile-installed Go tools (`protoc-gen-go@v1.28.1`,
|
||||||
`golangci-lint@v2.12.2`) by module hash, not mutable tag
|
`golangci-lint@v2.12.2`) by module hash, not mutable tag
|
||||||
- Set `make test` timeout to 30s (currently 10s)
|
- Set `make test` timeout to 30s (currently 10s)
|
||||||
- Add explicit README "Rationale" heading (content exists under other
|
- Add explicit README "Rationale" heading (content exists under other
|
||||||
names); name the author in the README Description first line
|
names); name the author in the README Description first line
|
||||||
- Reconcile root-level AGENTS.md with directory-hygiene policy (keep
|
- Reconcile root-level AGENTS.md with directory-hygiene policy (keep or
|
||||||
or relocate)
|
relocate)
|
||||||
- Add a `make build` target
|
- Add a `make build` target
|
||||||
- Rewrite `make hooks` to use printf or a heredoc instead of
|
- Rewrite `make hooks` to use printf or a heredoc instead of non-portable
|
||||||
non-portable `echo '...\n...'`
|
`echo '...\n...'`
|
||||||
- Answer the 14 owner design questions in the README 1.0 roadmap:
|
- Answer the 14 owner design questions in the README 1.0 roadmap:
|
||||||
- Format: simplify MFFileChecksum; store file mode; drop atime;
|
- Format: simplify MFFileChecksum; store file mode; drop atime; specify path
|
||||||
specify path normalization rules; version byte after magic;
|
normalization rules; version byte after magic; length-prefix after magic
|
||||||
length-prefix after magic
|
- Signatures: hash covers compressed or uncompressed data; sign raw bytes vs
|
||||||
- Signatures: hash covers compressed or uncompressed data; sign raw
|
hex canonical string; detached .mf.sig support; GPG subprocess vs pure-Go
|
||||||
bytes vs hex canonical string; detached .mf.sig support; GPG
|
crypto
|
||||||
subprocess vs pure-Go crypto
|
- Implementation: deterministic manifests by default; consolidate duplicate
|
||||||
- Implementation: deterministic manifests by default; consolidate
|
scanner/checker implementations; export the manifest type; canonical Go
|
||||||
duplicate scanner/checker implementations; export the manifest
|
module path for 1.0
|
||||||
type; canonical Go module path for 1.0
|
|
||||||
- Format and correctness:
|
- Format and correctness:
|
||||||
- Resolve proto go_package vs go.mod module path inconsistency
|
- Resolve proto go_package vs go.mod module path inconsistency
|
||||||
- Specify and validate path invariants (UTF-8, forward-slash,
|
- Specify and validate path invariants (UTF-8, forward-slash, relative, no
|
||||||
relative, no .., no leading /)
|
.., no leading /)
|
||||||
- Remove or deprecate atime; reserve mode field; add version byte
|
- Remove or deprecate atime; reserve mode field; add version byte (all
|
||||||
(all pending design answers)
|
pending design answers)
|
||||||
- Write a standalone format specification document
|
- Write a standalone format specification document
|
||||||
- Library:
|
- Library:
|
||||||
- Delete internal/scanner and internal/checker; consolidate on the
|
- Delete internal/scanner and internal/checker; consolidate on the mfer/
|
||||||
mfer/ package versions (pending design answer)
|
package versions (pending design answer)
|
||||||
- Add decompression size limit via io.LimitReader in
|
- Add decompression size limit via io.LimitReader in deserializeInner()
|
||||||
deserializeInner()
|
- Fix errors.Is dead code in checker; make AddFile verify totalRead == size
|
||||||
- Fix errors.Is dead code in checker; make AddFile verify
|
|
||||||
totalRead == size
|
|
||||||
- Export manifest type or define a public interface (pending)
|
- Export manifest type or define a public interface (pending)
|
||||||
- Replace GPG subprocess with pure-Go crypto (pending); add timeouts
|
- Replace GPG subprocess with pure-Go crypto (pending); add timeouts to
|
||||||
to remaining subprocess calls
|
remaining subprocess calls
|
||||||
- CLI:
|
- CLI:
|
||||||
- Kebab-case primary flag names; fix fetch URL construction with
|
- Kebab-case primary flag names; fix fetch URL construction with
|
||||||
url.JoinPath; add http.Client timeout and retry with backoff to
|
url.JoinPath; add http.Client timeout and retry with backoff to fetch;
|
||||||
fetch; rate-limit Checker progress output; add --deterministic
|
rate-limit Checker progress output; add --deterministic flag or default;
|
||||||
flag or default; wire top-level --version properly
|
wire top-level --version properly
|
||||||
- Testing:
|
- Testing:
|
||||||
- Fuzz NewManifestFromReader; end-to-end tests for freshen and fetch
|
- Fuzz NewManifestFromReader; end-to-end tests for freshen and fetch
|
||||||
- Documentation:
|
- Documentation:
|
||||||
- Promote docs/FORMAT.md as primary spec reference; audit error
|
- Promote docs/FORMAT.md as primary spec reference; audit error messages;
|
||||||
messages; document the signature scheme fully
|
document the signature scheme fully
|
||||||
- Release:
|
- Release:
|
||||||
- Finalize module path, bump version constant, SemVer --version
|
- Finalize module path, bump version constant, SemVer --version output, tag
|
||||||
output, tag v1.0.0
|
v1.0.0
|
||||||
|
|||||||
10
package.json
Normal file
10
package.json
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"name": "mfer",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"description": "Development tooling for the mfer repository: prettier, used by script/fmt and script/fmt-check to format and verify Markdown and JSON.",
|
||||||
|
"license": "WTFPL",
|
||||||
|
"devDependencies": {
|
||||||
|
"prettier": "3.9.6"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -130,9 +130,13 @@ main() {
|
|||||||
if missing make; then pkg_install gnumake make make make; fi
|
if missing make; then pkg_install gnumake make make make; fi
|
||||||
|
|
||||||
# ---- JS / docs repos ----
|
# ---- JS / docs repos ----
|
||||||
# ensure_node
|
# This is a Go repo, but node and yarn are required anyway: prettier
|
||||||
# ensure_yarn
|
# formats the Markdown and JSON, and script/fmt-check verifies it.
|
||||||
# install_js_deps
|
# The version is pinned by package.json/yarn.lock, whose integrity
|
||||||
|
# hashes --frozen-lockfile enforces.
|
||||||
|
ensure_node
|
||||||
|
ensure_yarn
|
||||||
|
install_js_deps
|
||||||
|
|
||||||
# ---- Go repos ----
|
# ---- Go repos ----
|
||||||
if missing go; then pkg_install go golang go go; fi
|
if missing go; then pkg_install go golang go go; fi
|
||||||
|
|||||||
@@ -2,7 +2,8 @@
|
|||||||
# script/fmt: format all files (writes).
|
# script/fmt: format all files (writes).
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
||||||
# (mirrors the old Makefile prerequisite; the generated file is
|
# (mirrors the old Makefile prerequisite; the generated file is
|
||||||
@@ -19,9 +20,8 @@ main() {
|
|||||||
ensure_pb
|
ensure_pb
|
||||||
gofumpt -l -w mfer internal cmd
|
gofumpt -l -w mfer internal cmd
|
||||||
golangci-lint run --fix
|
golangci-lint run --fix
|
||||||
# prettier is best-effort, as in the old Makefile (- prefix)
|
# Markdown and JSON, over the same file set script/fmt-check verifies.
|
||||||
prettier -w *.json || true
|
"$SCRIPT_DIR/prettier" --write
|
||||||
prettier -w *.md || true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
@@ -1,28 +1,14 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/fmt-check: check formatting (read-only). Same scope as
|
# script/fmt-check: check formatting (read-only). Same scope as
|
||||||
# script/fmt, but fails instead of writing.
|
# script/fmt, but fails instead of writing: Go via script/fmt-check-go,
|
||||||
|
# Markdown and JSON via script/prettier.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|
||||||
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
|
||||||
# (mirrors the old Makefile prerequisite; the generated file is
|
|
||||||
# committed, so this is normally a no-op).
|
|
||||||
ensure_pb() {
|
|
||||||
if [ ! -f mfer/mf.pb.go ] ||
|
|
||||||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
|
||||||
(cd mfer && go generate .)
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
"$SCRIPT_DIR/fmt-check-go"
|
||||||
ensure_pb
|
"$SCRIPT_DIR/prettier" --check
|
||||||
if [ -n "$(gofmt -l .)" ]; then
|
|
||||||
echo "gofmt: files need formatting:" >&2
|
|
||||||
gofmt -l . >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
29
script/fmt-check-go
Executable file
29
script/fmt-check-go
Executable file
@@ -0,0 +1,29 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/fmt-check-go: check Go formatting (read-only). Split out from
|
||||||
|
# script/fmt-check so the Docker lint stage, whose image has no node and
|
||||||
|
# therefore no prettier, can run the Go half on its own.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
||||||
|
# (mirrors the old Makefile prerequisite; the generated file is
|
||||||
|
# committed, so this is normally a no-op).
|
||||||
|
ensure_pb() {
|
||||||
|
if [ ! -f mfer/mf.pb.go ] ||
|
||||||
|
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
||||||
|
(cd mfer && go generate .)
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
ensure_pb
|
||||||
|
if [ -n "$(gofmt -l .)" ]; then
|
||||||
|
echo "gofmt: files need formatting:" >&2
|
||||||
|
gofmt -l . >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
69
script/prettier
Executable file
69
script/prettier
Executable file
@@ -0,0 +1,69 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/prettier: run prettier over this repo's canonical file set.
|
||||||
|
#
|
||||||
|
# Takes exactly one mode argument, --write or --check, and applies the
|
||||||
|
# same patterns in both modes. script/fmt and script/fmt-check both go
|
||||||
|
# through here, so the set of files that get formatted and the set that
|
||||||
|
# get verified cannot drift apart.
|
||||||
|
#
|
||||||
|
# Failures are never swallowed: a missing prettier is an error, not a
|
||||||
|
# silent skip. A formatter that quietly does nothing is worse than one
|
||||||
|
# that fails loudly.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "usage: script/prettier --write|--check" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
# Prefer the version pinned by package.json/yarn.lock so that CI and
|
||||||
|
# developer machines format identically. Fall back to a prettier on PATH,
|
||||||
|
# but say so, because a different version formats differently.
|
||||||
|
find_prettier() {
|
||||||
|
if [ -x "$ROOT/node_modules/.bin/prettier" ]; then
|
||||||
|
printf '%s\n' "$ROOT/node_modules/.bin/prettier"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if command -v prettier >/dev/null 2>&1; then
|
||||||
|
echo "prettier: node_modules/.bin/prettier is absent; using the" \
|
||||||
|
"prettier on PATH, which may be a different version than the" \
|
||||||
|
"one pinned in package.json. Run script/bootstrap to install" \
|
||||||
|
"the pinned version." >&2
|
||||||
|
command -v prettier
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
[ "$#" -eq 1 ] || usage
|
||||||
|
case "$1" in
|
||||||
|
--write | --check) mode="$1" ;;
|
||||||
|
*) usage ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
if ! prettier_bin="$(find_prettier)"; then
|
||||||
|
echo "prettier: not found." >&2
|
||||||
|
echo " Install it with: script/bootstrap" >&2
|
||||||
|
echo " (installs the version pinned in package.json/yarn.lock)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Markdown and JSON, repo-wide rather than root-only, so files in
|
||||||
|
# subdirectories (docs/, once it exists) are covered too. Exclusions
|
||||||
|
# live in .prettierignore; REPO_POLICIES.md is excluded there because
|
||||||
|
# it is a verbatim copy of an upstream document.
|
||||||
|
#
|
||||||
|
# --no-error-on-unmatched-pattern is deliberately NOT used: both
|
||||||
|
# patterns always match at least one tracked file (README.md,
|
||||||
|
# package.json), so an empty match means the glob broke, and prettier
|
||||||
|
# erroring out is exactly what we want rather than a vacuous pass.
|
||||||
|
"$prettier_bin" "$mode" "**/*.md"
|
||||||
|
"$prettier_bin" "$mode" "**/*.json"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
8
yarn.lock
Normal file
8
yarn.lock
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
||||||
|
# yarn lockfile v1
|
||||||
|
|
||||||
|
|
||||||
|
prettier@3.9.6:
|
||||||
|
version "3.9.6"
|
||||||
|
resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.9.6.tgz#b3ea5146515d40fc53f18aa63f74dfab1e10dbf6"
|
||||||
|
integrity sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==
|
||||||
Reference in New Issue
Block a user