1 Commits

Author SHA1 Message Date
clawbot
75c88d0a52 fix: use os.ErrNotExist instead of errors.New() for file-not-found detection in checkers
errors.Is(err, errors.New("file does not exist")) can never match because
errors.New creates a unique value each time. Replace with os.ErrNotExist
which is the standard Go sentinel for file-not-found errors.

Fixes both mfer/checker.go and internal/checker/checker.go.
2026-02-08 17:13:20 -08:00
51 changed files with 408 additions and 2378 deletions

23
.drone.yml Normal file
View File

@@ -0,0 +1,23 @@
kind: pipeline
name: test-docker-build
steps:
- name: test-docker-build
image: plugins/docker
network_mode: bridge
settings:
repo: sneak/mfer
build_args_from_env: [ DRONE_COMMIT_SHA ]
dry_run: true
custom_dns: [ 116.202.204.30 ]
tags:
- ${DRONE_COMMIT_SHA:0:7}
- ${DRONE_BRANCH}
- latest
- name: notify
image: plugins/slack
settings:
webhook:
from_secret: SLACK_WEBHOOK_URL
when:
event: pull_request

View File

@@ -1,9 +0,0 @@
name: check
on: [push]
jobs:
check:
runs-on: ubuntu-latest
steps:
# actions/checkout v4.2.2, 2026-03-16
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
- run: script/cibuild

8
.gitignore vendored
View File

@@ -3,11 +3,3 @@
*.tmp *.tmp
*.dockerimage *.dockerimage
/vendor /vendor
vendor.tzst
modcache.tzst
# Generated manifest files
.index.mf
# Stale files
.drone.yml

BIN
.index.mf Normal file

Binary file not shown.

View File

@@ -1,30 +0,0 @@
# Agent Instructions
Read `REPO_POLICIES.md` before making any changes. It is the authoritative
source for coding standards, formatting, linting, and workflow rules.
## Workflow
- When fixing a bug, write a failing test FIRST. Only after the test fails,
write the code to fix the bug. Then ensure the test passes. Leave the test in
place and commit it with the bugfix. Don't run shell commands to test bugfixes
or reproduce bugs. Write tests!
- After each change, run `make fmt`, then `make test`, then `make lint`. Fix any
failures before committing.
- After each change, commit only the files you've changed. Push after committing.
## Attribution
- Never mention Claude, Anthropic, or any AI/LLM tooling in commit messages. Do
not use attribution.
## Repository-Specific Notes
- This is a Go library + CLI tool for generating `.mf` manifest files.
- The proto definition is in `mfer/mf.proto`; generated `.pb.go` files are
committed (required for `go get` compatibility).
- The format specification is in `FORMAT.md`.
- See the TODO section in `README.md` for the 1.0 implementation plan
and open design questions.

20
CLAUDE.md Normal file
View File

@@ -0,0 +1,20 @@
# Important Rules
- when fixing a bug, write a failing test FIRST. only after the test fails, write
the code to fix the bug. then ensure the test passes. leave the test in
place and commit it with the bugfix. don't run shell commands to test
bugfixes or reproduce bugs. write tests!
- never, ever mention claude or anthropic in commit messages. do not use attribution
- after each change, run "make fmt".
- after each change, run "make test" and ensure all tests pass.
- after each change, run "make lint" and ensure no linting errors. fix any
you find, one by one.
- after each change, commit the files you've changed. push after
committing.
- NEVER use `git add -A`. always add only individual files that you've changed.

View File

@@ -1,38 +1,37 @@
# Lint stage — fast feedback on formatting and lint issues ################################################################################
# golangci/golangci-lint:v2.0.2 (2026-03-14) #2345678911234567892123456789312345678941234567895123456789612345678971234567898
FROM golangci/golangci-lint@sha256:d55581f7797e7a0877a7c3aaa399b01bdc57d2874d6412601a046cc4062cb62e AS lint ################################################################################
FROM sneak/builder:2022-12-08 AS builder
WORKDIR /src ENV DEBIAN_FRONTEND noninteractive
COPY go.mod go.sum ./ WORKDIR /build
RUN go mod download COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum /build/
COPY ./vendor.tzst /build/vendor.tzst
COPY . . COPY ./modcache.tzst /build/modcache.tzst
COPY ./internal ./internal
# Touch .pb.go so make does not try to regenerate via protoc (file is committed) COPY ./bin/gitrev.sh ./bin/gitrev.sh
RUN touch mfer/mf.pb.go COPY ./mfer ./mfer
COPY ./cmd ./cmd
RUN make fmt-check ARG GITREV unknown
RUN make lint ARG DRONE_COMMIT_SHA unknown
# Build stage — tests and compilation
# golang:1.23 (2026-03-14)
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
# Force BuildKit to run the lint stage by creating a stage dependency
COPY --from=lint /src/go.sum /dev/null
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
# Touch .pb.go so make does not try to regenerate via protoc (file is committed)
RUN touch mfer/mf.pb.go
RUN make test
RUN cd cmd/mfer && go build -tags urfave_cli_no_docs -o /mfer .
RUN mkdir -p "$(go env GOMODCACHE)" && cd "$(go env GOMODCACHE)" && \
zstdmt -d --stdout /build/modcache.tzst | tar xf - && \
rm /build/modcache.tzst && cd /build
RUN \
cd mfer && go generate . && cd .. && \
GOPACKAGESDEBUG=true golangci-lint run ./... && \
mkdir vendor && cd vendor && \
zstdmt -d --stdout /build/vendor.tzst | tar xf - && rm /build/vendor.tzst && \
cd .. && \
make mfer.cmd
RUN rm -rf /build/vendor && go mod vendor && tar -c . | zstdmt -19 > /src.tzst
################################################################################
#2345678911234567892123456789312345678941234567895123456789612345678971234567898
################################################################################
## final image
################################################################################
FROM scratch FROM scratch
COPY --from=builder /mfer /mfer # we put all the source into the final image for posterity, it's small
COPY --from=builder /src.tzst /src.tzst
COPY --from=builder /build/mfer.cmd /mfer
ENTRYPOINT ["/mfer"] ENTRYPOINT ["/mfer"]

143
FORMAT.md
View File

@@ -1,143 +0,0 @@
# .mf File Format Specification
Version 1.0
## Overview
An `.mf` file is a binary manifest that describes a directory tree of files,
including their paths, sizes, and cryptographic checksums. It supports
optional GPG signatures for integrity verification and optional timestamps
for metadata preservation.
## File Structure
An `.mf` file consists of two parts, concatenated:
1. **Magic bytes** (8 bytes): the ASCII string `ZNAVSRFG`
2. **Outer message**: a Protocol Buffers serialized `MFFileOuter` message
There is no length prefix or version byte between the magic and the protobuf
message. The protobuf message extends to the end of the file.
See [`mfer/mf.proto`](mfer/mf.proto) for exact field numbers and types.
## Outer Message (`MFFileOuter`)
The outer message contains:
| Field | Number | Type | Description |
| ----------------- | ------ | ---------------- | ------------------------------------------------------------------------ |
| `version` | 101 | enum | Must be `VERSION_ONE` (1) |
| `compressionType` | 102 | enum | Compression of `innerMessage`; must be `COMPRESSION_ZSTD` (1) |
| `size` | 103 | int64 | Uncompressed size of `innerMessage` (corruption detection) |
| `sha256` | 104 | bytes | SHA-256 hash of the **compressed** `innerMessage` (corruption detection) |
| `uuid` | 105 | bytes | Random v4 UUID; must match the inner message UUID |
| `innerMessage` | 199 | bytes | Zstd-compressed serialized `MFFile` message |
| `signature` | 201 | bytes (optional) | GPG signature (ASCII-armored or binary) |
| `signer` | 202 | bytes (optional) | Full GPG key ID of the signer |
| `signingPubKey` | 203 | bytes (optional) | Full GPG signing public key |
### SHA-256 Hash
The `sha256` field (104) covers the **compressed** `innerMessage` bytes.
This allows verifying data integrity before decompression.
## Compression
The `innerMessage` field is compressed with [Zstandard (zstd)](https://facebook.github.io/zstd/).
Implementations must enforce a decompression size limit to prevent
decompression bombs. The reference implementation limits decompressed size to
256 MB.
## Inner Message (`MFFile`)
After decompressing `innerMessage`, the result is a serialized `MFFile`
(referred to as the manifest):
| Field | Number | Type | Description |
| ----------- | ------ | --------------------- | ------------------------------------- |
| `version` | 100 | enum | Must be `VERSION_ONE` (1) |
| `files` | 101 | repeated `MFFilePath` | List of files in the manifest |
| `uuid` | 102 | bytes | Random v4 UUID; must match outer UUID |
| `createdAt` | 201 | Timestamp (optional) | When the manifest was created |
## File Entries (`MFFilePath`)
Each file entry contains:
| Field | Number | Type | Description |
| ---------- | ------ | ------------------------- | ----------------------------------- |
| `path` | 1 | string | Relative file path (see Path Rules) |
| `size` | 2 | int64 | File size in bytes |
| `hashes` | 3 | repeated `MFFileChecksum` | At least one hash required |
| `mimeType` | 301 | string (optional) | MIME type |
| `mtime` | 302 | Timestamp (optional) | Modification time |
| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) |
Field 304 (`atime`) has been removed from the specification. Access time is
volatile and non-deterministic; it is not useful for integrity verification.
## Path Rules
All `path` values must satisfy these invariants:
- **UTF-8**: paths must be valid UTF-8
- **Forward slashes**: use `/` as the path separator (never `\`)
- **Relative only**: no leading `/`
- **No parent traversal**: no `..` path segments
- **No empty segments**: no `//` sequences
- **No trailing slash**: paths refer to files, not directories
Implementations must validate these invariants when reading and writing
manifests. Paths that violate these rules must be rejected.
## Hash Format (`MFFileChecksum`)
Each checksum is a single `bytes multiHash` field containing a
[multihash](https://multiformats.io/multihash/)-encoded value. Multihash is
self-describing: the encoded bytes include a varint algorithm identifier
followed by a varint digest length followed by the digest itself.
The 1.0 implementation writes SHA-256 multihashes (`0x12` algorithm code).
Implementations must be able to verify SHA-256 multihashes at minimum.
## Signature Scheme
Signing is optional. When present, the signature covers a canonical string
constructed as:
```
ZNAVSRFG-<UUID>-<SHA256>
```
Where:
- `ZNAVSRFG` is the magic bytes string (literal ASCII)
- `<UUID>` is the hex-encoded UUID from the outer message
- `<SHA256>` is the hex-encoded SHA-256 hash from the outer message (covering compressed data)
Components are separated by hyphens. The signature is produced by GPG over
this canonical string and stored in the `signature` field of the outer
message.
## Deterministic Serialization
By default, manifests are generated deterministically:
- File entries are sorted by `path` in **lexicographic byte order**
- `createdAt` is omitted unless explicitly requested
- `atime` is never included (field removed from schema)
This ensures that two independent runs over the same directory tree produce
byte-identical `.mf` files (assuming file contents and metadata have not
changed).
## MIME Type
The recommended MIME type for `.mf` files is `application/octet-stream`.
The `.mf` file extension is the canonical identifier.
## Reference
- Proto definition: [`mfer/mf.proto`](mfer/mf.proto)
- Reference implementation: [git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)

View File

@@ -5,7 +5,7 @@ export PATH := $(PATH):$(GOPATH)/bin
PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
ARCH := $(shell uname -m) ARCH := $(shell uname -m)
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown) GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh)
APPNAME := mfer APPNAME := mfer
VERSION := 0.1.0 VERSION := 0.1.0
export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH) export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
@@ -13,24 +13,18 @@ GOLDFLAGS += -X main.Version=$(VERSION)
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD) GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
GOFLAGS := -ldflags "$(GOLDFLAGS)" GOFLAGS := -ldflags "$(GOLDFLAGS)"
.PHONY: bootstrap setup docker default run ci test check lint fmt fmt-check hooks fixme .PHONY: docker default run ci test fixme
default: fmt test default: fmt test
bootstrap:
@script/bootstrap
setup:
@script/setup
run: ./bin/mfer run: ./bin/mfer
./$< ./$<
./$< gen ./$< gen
ci: test ci: test
test: test: $(SOURCEFILES) mfer/mf.pb.go
@script/test go test -v --timeout 10s ./...
$(PROTOC_GEN_GO): $(PROTOC_GEN_GO):
test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
@@ -38,17 +32,8 @@ $(PROTOC_GEN_GO):
fixme: fixme:
@grep -nir fixme . | grep -v Makefile @grep -nir fixme . | grep -v Makefile
check:
@script/check
fmt-check:
@script/fmt-check
hooks:
@script/install-precommit
devprereqs: devprereqs:
which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v2.0.2 which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest
mfer/mf.pb.go: mfer/mf.proto mfer/mf.pb.go: mfer/mf.proto
cd mfer && go generate . cd mfer && go generate .
@@ -60,14 +45,17 @@ bin/mfer: $(SOURCEFILES) mfer/mf.pb.go
clean: clean:
rm -rfv mfer/*.pb.go bin/mfer cmd/mfer/mfer *.dockerimage rm -rfv mfer/*.pb.go bin/mfer cmd/mfer/mfer *.dockerimage
fmt: fmt: mfer/mf.pb.go
@script/fmt gofumpt -l -w mfer internal cmd
golangci-lint run --fix
-prettier -w *.json
-prettier -w *.md
lint: lint:
@script/lint golangci-lint run
sh -c 'test -z "$$(gofmt -l .)"'
docker: docker: sneak-mfer.$(ARCH).tzst.dockerimage
@script/docker
sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer . docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .

291
README.md
View File

@@ -23,41 +23,38 @@ itch in 2022 and is currently a one-person effort, though the goal is for
this to emerge as a de-facto standard and be incorporated into other this to emerge as a de-facto standard and be incorporated into other
software. A compatible javascript library is planned. software. A compatible javascript library is planned.
# Phases
Manifest generation happens in two distinct phases:
## Phase 1: Enumeration
Walking directories and calling `stat()` on files to collect metadata (path, size, mtime, ctime). This builds the list of files to be scanned. Relatively fast as it only reads filesystem metadata, not file contents.
**Progress:** `EnumerateStatus` with `FilesFound` and `BytesFound`
## Phase 2: Scan (ToManifest)
Reading file contents and computing cryptographic hashes for manifest generation. This is the expensive phase that reads all file data from disk.
**Progress:** `ScanStatus` with `TotalFiles`, `ScannedFiles`, `TotalBytes`, `ScannedBytes`, `BytesPerSec`
# Code Conventions
- **Logging:** Never use `fmt.Printf` or write to stdout/stderr directly in normal code. Use the `internal/log` package for all output (`log.Info`, `log.Infof`, `log.Debug`, `log.Debugf`, `log.Progressf`, `log.ProgressDone`).
- **Filesystem abstraction:** Use `github.com/spf13/afero` for filesystem operations to enable testing and flexibility.
- **CLI framework:** Use `github.com/urfave/cli/v2` for command-line interface.
- **Serialization:** Use Protocol Buffers for manifest file format.
- **Internal packages:** Non-exported implementation details go in `internal/` subdirectories.
- **Concurrency:** Use `sync.RWMutex` for protecting shared state; prefer channels for progress reporting.
- **Progress channels:** Use buffered channels (size 1) with non-blocking sends to avoid blocking the main operation if the consumer is slow.
- **Context support:** Long-running operations should accept `context.Context` for cancellation.
- **NO_COLOR:** Respect the `NO_COLOR` environment variable for disabling colored output.
- **Options pattern:** Use `NewWithOptions(opts *Options)` constructor pattern for configurable types.
# Build Status # Build Status
CI runs via `script/cibuild` (`docker build .`), which executes `make [![Build Status](https://drone.datavi.be/api/badges/sneak/mfer/status.svg)](https://drone.datavi.be/sneak/mfer)
check` (formatting, linting, tests). The `main` branch must always be
green.
# Entrypoints
This repository adheres to the
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
standard: normalized scripts in `script/` are the entrypoints for the
development workflow, and the Makefile targets are thin shims that call
them. We provide:
- `script/bootstrap` — install all dependencies (Go, golangci-lint, Go
module download), idempotently
- `script/setup` — make a fresh clone ready for development: runs
`script/bootstrap`, then `script/install-precommit`
- `script/projectname` — output the project name (`mfer`); used by other
scripts such as `script/docker`
- `script/test` — run the test suite (`go test`), regenerating the
protobuf code first if it is stale
- `script/lint` — run `golangci-lint` and verify `gofmt` cleanliness
- `script/fmt` — format all code and docs (writes): `gofumpt`,
`golangci-lint run --fix`, and prettier for JSON/Markdown
- `script/fmt-check` — check formatting without writing
- `script/check` — run `script/test`, `script/lint`, and
`script/fmt-check`
- `script/docker` — build the Docker image tagged with the project name
- `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile
runs the checks)
- `script/precommit` — pre-commit checks: `go mod tidy` verification,
then `script/check`
- `script/install-precommit` — install the git pre-commit hook that
runs `script/precommit`
# Participation # Participation
@@ -74,9 +71,6 @@ requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
Any changes submitted to this project must also be Any changes submitted to this project must also be
[WTFPL-licensed](https://wtfpl.net) to be considered. [WTFPL-licensed](https://wtfpl.net) to be considered.
See [`REPO_POLICIES.md`](REPO_POLICIES.md) for detailed coding standards,
tooling requirements, and workflow conventions.
# Problem Statement # Problem Statement
Given a plain URL, there is no standard way to safely and programmatically Given a plain URL, there is no standard way to safely and programmatically
@@ -154,6 +148,10 @@ The manifest file would do several important things:
- metadata size should not be used as an excuse to sacrifice utility (such - metadata size should not be used as an excuse to sacrifice utility (such
as providing checksums over each chunk of a large file) as providing checksums over each chunk of a large file)
# Limitations
- **Manifest size:** Manifests must fit entirely in system memory during reading and writing.
# Open Questions # Open Questions
- Should the manifest file include checksums of individual file chunks, or just for the whole assembled file? - Should the manifest file include checksums of individual file chunks, or just for the whole assembled file?
@@ -241,227 +239,6 @@ regardless of filesystem format.
Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your
desired username for an account on this Gitea instance. desired username for an account on this Gitea instance.
# TODO: Remaining Work for 1.0
## Design Questions (Owner Decision Required)
These require @sneak's input before implementation. Answers should be added
inline below each question.
### Format Design
**1. Should `MFFileChecksum` be simplified?** Currently it's a separate
message wrapping a single `bytes multiHash` field. Since multihash
already self-describes the algorithm, `repeated bytes hashes` directly on
`MFFilePath` would be simpler and reduce per-file protobuf overhead. Is
the extra message layer intentional (e.g. planning to add per-hash
metadata like `verified_at`)?
> _answer:_
**2. Should file permissions/mode be stored?** The format stores
mtime/ctime but not Unix file permissions. For archival use this may not
matter, but for software distribution or filesystem restoration it's a
gap. Should we reserve a field now (e.g. `optional uint32 mode = 305`)
even if we don't populate it yet?
> _answer:_
**3. Should `atime` be removed from the schema?** Access time is
volatile, non-deterministic, and often disabled (`noatime`). Including it
means two manifests of the same directory at different times will differ,
which conflicts with the determinism goal. Remove it, or document it as
"never set by default"?
> _answer:_
**4. What are the path normalization rules?** The proto has `string path`
with no specification about: always forward-slash? Must be relative? No
`..` components allowed? UTF-8 NFC vs NFD normalization (macOS vs
Linux)? Max path length? This is a security issue (path traversal) and a
cross-platform compatibility issue. What rules should the spec mandate?
> _answer:_
**5. Should we add a version byte after the magic?** Currently
`ZNAVSRFG` is followed immediately by protobuf. Adding a version byte
(`ZNAVSRFG\x01`) would allow future framing changes without requiring
protobuf parsing to detect the version. `MFFileOuter.Version` serves
this purpose but requires successful deserialization to read. Worth the
extra byte?
> _answer:_
**6. Should we add a length-prefix after the magic?** Protobuf is not
self-delimiting. If we ever want to concatenate manifests or append data
after the protobuf, the current framing is insufficient. Add a varint or
fixed-width length-prefix?
> _answer:_
### Signature Design
**7. What does the outer SHA-256 hash cover — compressed or uncompressed
data?** The code currently hashes compressed data (good for verifying
before decompression), but this should be explicitly documented. Which is
the intended behavior?
> _answer:_
**8. Should `signatureString()` sign raw bytes instead of a hex-encoded
string?** Currently the canonical string is `MAGIC-UUID-MULTIHASH` with
hex encoding, which adds a transformation layer. Signing the raw `sha256`
bytes (or compressed `innerMessage` directly) would be simpler. Keep the
string format or switch to raw bytes?
> _answer:_
**9. Should we support detached signature files (`.mf.sig`)?** Embedded
signatures are better for single-file distribution. Detached `.mf.sig`
files follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are
simpler for HTTP serving. Support both modes?
> _answer:_
**10. GPG vs pure-Go crypto for signatures?** Shelling out to `gpg` is
fragile (may not be installed, version-dependent output).
`github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could
use Ed25519/signify (simpler, no key management). Which direction?
> _answer:_
### Implementation Design
**11. Should manifests be deterministic by default?** This means: sort
file entries by path, omit `createdAt` timestamp (or make it opt-in), no
`atime`. Should determinism be the default, with a
`--include-timestamps` flag to opt in?
> _answer:_
**12. Should we consolidate or keep both scanner/checker
implementations?** There are two parallel implementations:
`mfer/scanner.go` + `mfer/checker.go` (typed with `FileSize`,
`RelFilePath`) and `internal/scanner/` + `internal/checker/` (raw
`int64`, `string`). The `mfer/` versions are superior. Delete the
`internal/` versions?
> _answer:_
**13. Should the `manifest` type be exported?** Currently unexported with
exported constructors (`NewManifestFromReader`, `NewManifestFromFile`).
Consumers can't declare `var m *mfer.manifest`. Export the type, or
define an interface?
> _answer:_
**14. What should the Go module path be for 1.0?** Currently
`sneak.berlin/go/mfer` in `go.mod` but `git.eeqj.de/sneak/mfer/mfer` in
the proto `go_package` option. Which is canonical?
> _answer:_
## Implementation Tasks
### Repo Infrastructure
- [ ] Add `.golangci.yml` (fetch from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`)
- [ ] Add `.editorconfig`
- [ ] Add `.gitea/workflows/check.yml` that runs `docker build .`
### Format & Correctness
- [ ] Resolve proto `go_package` path inconsistency
(`git.eeqj.de/sneak/mfer/mfer` vs `sneak.berlin/go/mfer`)
- [ ] Specify path invariants — add proto comments requiring UTF-8,
forward-slash, relative paths, no `..`, no leading `/`; validate
in `Builder.AddFile` and `Builder.AddFileWithHash` (pending design
question answer)
- [ ] Remove or deprecate `atime` from proto (pending design question
answer)
- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future
file permissions (pending design question answer)
- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version
1 (pending design question answer)
- [ ] Write format specification document — separate from README:
magic, outer structure, compression, inner structure, path
invariants, signature scheme, canonical serialization
### Library
- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate
on `mfer/` package versions; update CLI code (pending design
question answer)
- [ ] Add deterministic file ordering — sort entries by path
(lexicographic, byte-order) in `Builder.Build()`; add test
asserting byte-identical output from two runs
- [ ] Add decompression size limit — `io.LimitReader` in
`deserializeInner()` with `m.pbOuter.Size` as bound
- [ ] Fix `errors.Is` dead code in checker — replace with
`os.IsNotExist(err)` or `errors.Is(err, fs.ErrNotExist)`
- [ ] Fix `AddFile` to verify size — check `totalRead == size` after
reading, return error on mismatch
- [ ] Export the `manifest` type or define a public interface (pending
design question answer) — currently consumers cannot hold a reference
to a loaded manifest in their own type declarations
- [ ] Replace GPG subprocess calls with pure-Go crypto (pending design
question answer) — current implementation shells out to `gpg` which
may not be installed
- [ ] Add timeout to any remaining subprocess calls
### CLI
- [ ] Fix flag naming — all CLI flags should use kebab-case as primary
(`--include-dotfiles`, `--follow-symlinks`)
- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or
`url.JoinPath()` instead of string concatenation
- [ ] Add progress rate-limiting to Checker — throttle to once per
second, matching Scanner
- [ ] Add `--deterministic` flag or make it default — omit `createdAt`,
sort files (pending design question answer)
- [ ] Wire `--version` flag properly (currently only a `version`
subcommand exists; top-level `--version` shows urfave/cli generic
output)
- [ ] Add retry logic to `fetch` — currently no retries on transient
HTTP errors; needs exponential backoff
- [ ] `fetch` command uses bare `http.Get` with no timeout — needs
`http.Client` with configurable timeout
### Testing & Robustness
- [ ] Add fuzzing tests for `NewManifestFromReader` — protobuf
deserialization of untrusted input needs fuzz coverage
- [ ] Add integration test for `freshen` CLI command — current tests
only verify setup, not the actual freshen operation end-to-end
- [ ] Add test for `fetch` CLI command end-to-end (currently only
`downloadFile` is tested)
### Documentation
- [ ] Promote `FORMAT.md` as primary spec reference; README should link
to it more prominently
- [ ] Audit and update all error messages for consistency and
helpfulness
- [ ] Document the signature scheme more thoroughly (canonical string
format, verification steps)
### Release
- [ ] Finalize Go module path
- [ ] Update version constant in `mfer/constants.go`
- [ ] Add `--version` output matching SemVer
- [ ] Tag `v1.0.0`
# See Also
## Prior Art: Metalink
- [Metalink - Mozilla Wiki](https://wiki.mozilla.org/Metalink)
- [Metalink - Wikipedia](https://en.wikipedia.org/wiki/Metalink)
- [RFC 5854 - The Metalink Download Description Format](https://datatracker.ietf.org/doc/html/rfc5854)
- [RFC 6249 - Metalink/HTTP: Mirrors and Hashes](https://www.rfc-editor.org/rfc/rfc6249.html)
## Links ## Links
- Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) - Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)

View File

@@ -1,408 +0,0 @@
---
title: Repository Policies
last_modified: 2026-07-06
---
This document covers repository structure, tooling, and workflow standards. Code
style conventions are in separate documents:
- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md)
(general, bash, Docker)
- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md)
- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
---
- Cross-project documentation (such as this file) must include
`last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync
with the authoritative source as policies evolve.
- **ALL external references must be pinned by cryptographic hash.** This
includes Docker base images, Go modules, npm packages, GitHub Actions, and
anything else fetched from a remote source. Version tags (`@v4`, `@latest`,
`:3.21`, etc.) are server-mutable and therefore remote code execution
vulnerabilities. The ONLY acceptable way to reference an external dependency
is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm
integrity hash in lockfile, GitHub Actions `@<commit-sha>`). No exceptions.
This also means never `curl | bash` to install tools like pyenv, nvm, rustup,
etc. Instead, download a specific release archive from GitHub, verify its hash
(hardcoded in the Dockerfile or script), and only then install. Unverified
install scripts are arbitrary remote code execution. This is the single most
important rule in this document. Double-check every external reference in
every file before committing. There are zero exceptions to this rule.
- Every repo with software must have a root `Makefile` with these targets:
`make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes),
`make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`),
`make docker`, and `make hooks` (installs pre-commit hook). A model Makefile
is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`.
- Repos follow the
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
pattern: the implementation of each Makefile target lives in an executable
script in `script/` (`script/bootstrap`, `script/setup`, `script/test`,
`script/lint`, `script/fmt`, `script/fmt-check`, `script/check`,
`script/docker`), and the Makefile targets are thin shims that call them. The
scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in
minimal containers (e.g. alpine images have no bash); locate the repo root
with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From
the standard's canonical set we use `bootstrap`, `setup` (make the repo ready
for development after a fresh clone: runs `bootstrap`, then
`install-precommit`, plus any repo-specific initialization), `test`, and
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
assumes nothing is present: base tools come from nix, apt, brew, or apk
(detected in that order; apt runs noninteractive). For node it uses the
installed node if present; otherwise it installs a PINNED node version via
nvm, first installing nvm itself if missing — from a hash-verified GitHub
release archive (never `curl | sh`), with bash installed as an explicit
prerequisite since nvm requires bash. yarn is then pinned via
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
always exact versions. `script/cibuild` runs the CI build: it changes to the
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
scripts are our own extensions to the standard: `script/check` runs
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
what the git pre-commit hook runs, and it calls `script/check`;
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
target shims to it); and `script/projectname` (literally that filename) simply
outputs the project's name. Scripts that need the name call
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
so those scripts stay byte-identical across all repos. Repo-type-specific
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
`script/precommit`, not in the hook itself. Model scripts are at
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
must document the provided scripts in an **Entrypoints** section (see the
README requirements below).
- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.)
instead of invoking the underlying tools directly. The Makefile is the single
source of truth for how these operations are run.
- The Makefile is authoritative documentation for how the repo is used. Beyond
the required targets above, it should have targets for every common operation:
running a local development server (`make run`, `make dev`), re-initializing
or migrating the database (`make db-reset`, `make migrate`), building
artifacts (`make build`), generating code, seeding data, or anything else a
developer would do regularly. If someone checks out the repo and types
`make<tab>`, they should see every meaningful operation available. A new
contributor should be able to understand the entire development workflow by
reading the Makefile.
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
as a build step so the build fails if the branch is not green. For non-server
repos, the Dockerfile should bring up a development environment and run
`make check`. For server repos, `make check` should run as an early build
stage before the final image is assembled. Dockerfiles install development
prerequisites by running `script/bootstrap` rather than duplicating installs
inline; COPY `script/` and the dependency manifests (`package.json` +
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
layer stays cached until dependencies change.
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
repos use a multistage build where linting runs in an independent stage based
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
`make fmt-check` and `make lint` before the full build begins. The build stage
then declares an explicit dependency on the lint stage via
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
linting before proceeding to compilation and tests. This ensures lint failures
surface in seconds rather than minutes, without blocking on dependency
download or compilation in the build stage.
The standard pattern for a Go repo Dockerfile is:
```dockerfile
# Lint stage — fast feedback on formatting and lint issues
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
FROM golangci/golangci-lint@sha256:... AS lint
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN make fmt-check
RUN make lint
# Build stage
# golang:1.x-alpine, YYYY-MM-DD
FROM golang@sha256:... AS builder
WORKDIR /src
# Force BuildKit to run the lint stage before proceeding
COPY --from=lint /src/go.sum /dev/null
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN make test
ARG VERSION=dev
RUN CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X main.Version=${VERSION}" \
-o /app ./cmd/app/
# Runtime stage
FROM alpine@sha256:...
COPY --from=builder /app /usr/local/bin/app
ENTRYPOINT ["app"]
```
Key points:
- The lint stage uses the `golangci/golangci-lint` image directly (it
includes both Go and the linter), so there is no need to install the
linter separately.
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
a stage dependency. BuildKit runs stages in parallel by default; without
this line, the build stage would not wait for lint to finish and a lint
failure might not fail the overall build.
- If the project uses `//go:embed` directives that reference build artifacts
(e.g. a web frontend compiled in a separate stage), the lint stage must
create placeholder files so the embed directives resolve. Example:
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
The lint stage should not depend on the actual build output — it exists to
fail fast.
- If the project requires CGO or system libraries for linting (e.g.
`vips-dev`), install them in the lint stage with `apk add`.
- The build stage runs `make test` after compilation setup. Tests run in the
build stage, not the lint stage, because they may require compiled
artifacts or heavier dependencies.
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
runs `script/cibuild` (which runs `docker build .`) on push. Since the
Dockerfile already runs `make check`, a successful build implies all checks
pass.
- Use platform-standard formatters: `black` for Python, `prettier` for
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
two exceptions: four-space indents (except Go), and `proseWrap: always` for
Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown,
HTML, CSS) should also have `.prettierrc` and `.prettierignore`.
- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local
testing is not possible in the repo, `script/precommit` may skip `script/test`
and run only `script/lint` and `script/fmt-check`. The hook is installed by
`script/install-precommit`; the Makefile must provide a `make hooks` target
that shims to it.
- All repos with software must have tests that run via the platform-standard
test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful
tests exist yet, add the most minimal test possible — e.g. importing the
module under test to verify it compiles/parses. There is no excuse for
`make test` to be a no-op.
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
Makefile.
- **`make test` should use the conditional verbose rerun pattern.** Run tests
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
show full output. This keeps CI logs and `docker build` output clean on
success (just package/suite summaries) while providing full diagnostic detail
on failure (every test case, every assertion). The general shell pattern:
```makefile
test:
@<test-command> || \
{ echo "--- Rerunning with -v for details ---"; \
<test-command-with-v>; exit 1; }
```
Go example:
```makefile
test:
@go test -timeout 30s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -timeout 30s -race -v ./...; exit 1; }
```
Python example:
```makefile
test:
@python -m pytest || \
{ echo "--- Rerunning with -v for details ---"; \
python -m pytest -v; exit 1; }
```
The `exit 1` ensures the target always fails after a rerun — the first run
already proved the tests are broken, so the build must not pass even if a
flaky test happens to succeed on the second attempt. The rerun exists solely
for diagnostic output.
- Docker builds must complete in under 5 minutes.
- `make check` must not modify any files in the repo. Tests may use temporary
directories.
- `main` must always pass `make check`, no exceptions.
- Never commit secrets. `.env` files, credentials, API keys, and private keys
must be in `.gitignore`. No exceptions.
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
Fetch the standard `.gitignore` from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
a new repo.
- **No build artifacts in version control.** Code-derived data (compiled
bundles, minified output, generated assets) must never be committed to the
repository if it can be avoided. The build process (e.g. Dockerfile, Makefile)
should generate these at build time. Notable exception: Go protobuf generated
files (`.pb.go`) ARE committed because repos need to work with `go get`, which
downloads code but does not execute code generation.
- Never use `git add -A` or `git add .`. Always stage files explicitly by name.
- Never force-push to `main`.
- Make all changes on a feature branch. You can do whatever you want on a
feature branch.
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
manually by the user. Fetch from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
- When pinning images or packages by hash, add a comment above the reference
with the version and date (YYYY-MM-DD).
- Use `yarn`, not `npm`.
- Write all dates as YYYY-MM-DD (ISO 8601).
- Simple projects should be configured with environment variables.
- Dockerized web services listen on port 8080 by default, overridable with
`PORT`.
- **HTTP/web services must be hardened for production internet exposure before
tagging 1.0.** This means full compliance with security best practices
including, without limitation, all of the following:
- **Security headers** on every response:
- `Strict-Transport-Security` (HSTS) with `max-age` of at least one year
and `includeSubDomains`.
- `Content-Security-Policy` (CSP) with a restrictive default policy
(`default-src 'self'` as a baseline, tightened per-resource as
needed). Never use `unsafe-inline` or `unsafe-eval` unless
unavoidable, and document the reason.
- `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required).
Prefer the `frame-ancestors` CSP directive as the primary control.
- `X-Content-Type-Options: nosniff`.
- `Referrer-Policy: strict-origin-when-cross-origin` (or stricter).
- `Permissions-Policy` restricting access to browser features the
application does not use (camera, microphone, geolocation, etc.).
- **Request and response limits:**
- Maximum request body size enforced on all endpoints (e.g. Go
`http.MaxBytesReader`). Choose a sane default per-route; never accept
unbounded input.
- Maximum response body size where applicable (e.g. paginated APIs).
- `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend
against slowloris attacks.
- `WriteTimeout` on the `http.Server`.
- `IdleTimeout` on the `http.Server`.
- Per-handler execution time limits via `context.WithTimeout` or
chi/stdlib `middleware.Timeout`.
- **Authentication and session security:**
- Rate limiting on password-based authentication endpoints. API keys are
high-entropy and not susceptible to brute force, so they are exempt.
- CSRF tokens on all state-mutating HTML forms. API endpoints
authenticated via `Authorization` header (Bearer token, API key) are
exempt because the browser does not attach these automatically.
- Passwords stored using bcrypt, scrypt, or argon2 — never plain-text,
MD5, or SHA.
- Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or
`Strict`) attributes.
- **Reverse proxy awareness:**
- True client IP detection when behind a reverse proxy
(`X-Forwarded-For`, `X-Real-IP`). The application must accept
forwarded headers only from a configured set of trusted proxy
addresses — never trust `X-Forwarded-For` unconditionally.
- **CORS:**
- Authenticated endpoints must restrict `Access-Control-Allow-Origin` to
an explicit allowlist of known origins. Wildcard (`*`) is acceptable
only for public, unauthenticated read-only APIs.
- **Error handling:**
- Internal errors must never leak stack traces, SQL queries, file paths,
or other implementation details to the client. Return generic error
messages in production; detailed errors only when `DEBUG` is enabled.
- **TLS:**
- Services never terminate TLS directly. They are always deployed behind
a TLS-terminating reverse proxy. The service itself listens on plain
HTTP. However, HSTS headers and `Secure` cookie flags must still be
set by the application so that the browser enforces HTTPS end-to-end.
This list is non-exhaustive. Apply defense-in-depth: if a standard security
hardening measure exists for HTTP services and is not listed here, it is
still expected. When in doubt, harden.
- `README.md` is the primary documentation. Required sections:
- **Description**: First line must include the project name, purpose,
category (web server, SPA, CLI tool, etc.), license, and author. Example:
"µPaaS is an MIT-licensed Go web application by @sneak that receives
git-frontend webhooks and deploys applications via Docker in realtime."
- **Getting Started**: Copy-pasteable install/usage code block.
- **Entrypoints**: Opens by stating that the repo adheres to the
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
standard (with that link), then documents each provided `script/`
entrypoint and its purpose.
- **Rationale**: Why does this exist?
- **Design**: How is the program structured?
- **TODO**: Update meticulously, even between commits. When planning, put
the todo list in the README so a new agent can pick up where the last one
left off.
- **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a
`LICENSE` file in the repo root and a License section in the README.
- **Author**: [@sneak](https://sneak.berlin).
- First commit of a new repo should contain only `README.md`.
- Go module root: `sneak.berlin/go/<name>`. Always run `go mod tidy` before
committing.
- Use SemVer.
- Database migrations live in `internal/db/migrations/` and must be embedded in
the binary.
- `000_migration.sql` — contains ONLY the creation of the migrations
tracking table itself. Nothing else.
- `001_schema.sql` — the full application schema.
- **Pre-1.0.0:** never add additional migration files (002, 003, etc.).
There is no installed base to migrate. Edit `001_schema.sql` directly.
- **Post-1.0.0:** add new numbered migration files for each schema change.
Never edit existing migrations after release.
- All repos should have an `.editorconfig` enforcing the project's indentation
settings.
- Avoid putting files in the repo root unless necessary. Root should contain
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
language-specific config). Everything else goes in a subdirectory. Canonical
subdirectory names:
- `bin/` — executable scripts and tools
- `cmd/` — Go command entrypoints
- `configs/` — configuration templates and examples
- `deploy/` — deployment manifests (k8s, compose, terraform)
- `docs/` — documentation and markdown (README.md stays in root)
- `internal/` — Go internal packages
- `internal/db/migrations/` — database migrations
- `pkg/` — Go library packages
- `share/` — systemd units, data files
- `static/` — static assets (images, fonts, etc.)
- `web/` — web frontend source
- When setting up a new repo, files from the `prompts` repo may be used as
templates. Fetch them from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/<path>`.
- New repos must contain at minimum:
- `README.md`, `.git`, `.gitignore`, `.editorconfig`
- `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo)
- `Makefile`
- `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`,
`lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
`install-precommit`)
- `Dockerfile`, `.dockerignore`
- `.gitea/workflows/check.yml`
- Go: `go.mod`, `go.sum`, `.golangci.yml`
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
- Python: `pyproject.toml`

211
TODO.md
View File

@@ -1,107 +1,122 @@
# Workflow # TODO: mfer 1.0
- branch (from `main`) ## Design Questions
- do the work in Next Step
- move Next Step to the top of Completed Steps
- move the top item of Future Steps into Next Step
- commit (`TODO.md` changes in the same commit as the work)
- merge to `main` if the branch is not protected, otherwise open a PR
- push
# Status *sneak: please answer inline below each question. These are preserved for posterity.*
pre-1.0. No git tags. README section "TODO: Remaining Work for 1.0" lists ### Format Design
open design questions and implementation tasks; policy compliance work is
in flight and unmerged.
# Next Step **1. Should `MFFileChecksum` be simplified?**
Currently it's a separate message wrapping a single `bytes multiHash` field. Since multihash already self-describes the algorithm, `repeated bytes hashes` directly on `MFFilePath` would be simpler and reduce per-file protobuf overhead. Is the extra message layer intentional (e.g. planning to add per-hash metadata like `verified_at`)?
Land the in-flight compliance branch chore/align-repo-policies: finish and > *answer:*
commit the uncommitted work (32 modified Go files, new untracked
.golangci.yml and TODO.md), confirm `make check` is green, merge the branch
(one commit ahead of main as of 2026-07-03) to main, and push.
# Completed Steps **2. Should file permissions/mode be stored?**
The format stores mtime/ctime but not Unix file permissions. For archival use (ExFAT, filesystem-independent checksums) this may not matter, but for software distribution or filesystem restoration it's a gap. Should we reserve a field now (e.g. `optional uint32 mode = 305`) even if we don't populate it yet?
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, > *answer:*
Makefile shims, README Entrypoints section
- 2026-07-03: aligned repo tooling, docs, and config with standardized
policies (7d9a138, on chore/align-repo-policies, unmerged)
- 2026-06-28: moved to standardized repo policies (#56, on main)
- 2026-04-07: added 1.0 roadmap as README TODO section, removed old
TODO.md (#54)
- 2026-03-20: added Gitea Actions CI workflow (#53)
- 2026-03-17: added REPO_POLICIES.md, renamed CLAUDE.md to AGENTS.md (#51);
removed committed .index.mf (#52)
- 2026-03-15: split Dockerfile with pre-built golangci-lint stage for
faster CI (#45)
- 2026-03-01: 1.0 quality polish: code review, tests, bug fixes, docs (#32)
- 2026-02-20: deterministic file ordering in Builder.Build() (#28);
removed committed vendor/modcache archives (#35)
- 2026-02-08: added --seed flag for deterministic manifest UUID
# Future Steps **3. Should `atime` be removed from the schema?**
Access time is volatile, non-deterministic, and often disabled (`noatime`). Including it means two manifests of the same directory at different times will differ, which conflicts with the determinism goal. Remove it, or document it as "never set by default"?
- Compliance (fold of TODO.md audit 2026-07-02; verify which items the > *answer:*
in-flight branch already closes, then check off):
- Add .editorconfig (canonical copy from sneak/prompts) **4. What are the path normalization rules?**
- Add standardized .golangci.yml (present untracked on the branch; The proto has `string path` with no specification about: always forward-slash? Must be relative? No `..` components allowed? UTF-8 NFC vs NFD normalization (macOS vs Linux)? Max path length? This is a security issue (path traversal) and a cross-platform compatibility issue. What rules should the spec mandate?
user-owned, copy verbatim)
- Make .gitignore cover secrets (.env, _.key, _.pem), OS files > *answer:*
(.DS_Store), and editor files (_.swp, _~)
- Make fmt-check/lint verify with gofumpt, not gofmt -l, so **5. Should we add a version byte after the magic?**
`make check` matches what `make fmt` writes Currently `ZNAVSRFG` is followed immediately by protobuf. Adding a version byte (`ZNAVSRFG\x01`) would allow future framing changes without requiring protobuf parsing to detect the version. `MFFileOuter.Version` serves this purpose but requires successful deserialization to read. Worth the extra byte?
- Add README "Getting Started" section with copy-pasteable
install/usage block > *answer:*
- Move FORMAT.md from repo root to docs/ and update the AGENTS.md
reference **6. Should we add a length-prefix after the magic?**
- Pin Makefile-installed Go tools (protoc-gen-go@v1.28.1, Protobuf is not self-delimiting. If we ever want to concatenate manifests or append data after the protobuf, the current framing is insufficient. Add a varint or fixed-width length-prefix?
golangci-lint@v2.0.2) by module hash, not mutable tag
- Set `make test` timeout to 30s (currently 10s) > *answer:*
- Add explicit README "Rationale" heading (content exists under other
names); name the author in the README Description first line ### Signature Design
- Reconcile root-level AGENTS.md with directory-hygiene policy (keep
or relocate) **7. What does the outer SHA-256 hash cover — compressed or uncompressed data?**
- Add a `make build` target The review notes it currently hashes compressed data (good for verifying before decompression), but this should be explicitly documented. Which is the intended behavior?
- Rewrite `make hooks` to use printf or a heredoc instead of
non-portable `echo '...\n...'` > *answer:*
- Answer the 14 owner design questions in the README 1.0 roadmap:
- Format: simplify MFFileChecksum; store file mode; drop atime; **8. Should `signatureString()` sign raw bytes instead of a hex-encoded string?**
specify path normalization rules; version byte after magic; Currently the canonical string is `MAGIC-UUID-MULTIHASH` with hex encoding, which adds a transformation layer. Signing the raw `sha256` bytes (or compressed `innerMessage` directly) would be simpler. Keep the string format or switch to raw bytes?
length-prefix after magic
- Signatures: hash covers compressed or uncompressed data; sign raw > *answer:*
bytes vs hex canonical string; detached .mf.sig support; GPG
subprocess vs pure-Go crypto **9. Should we support detached signature files (`.mf.sig`)?**
- Implementation: deterministic manifests by default; consolidate Embedded signatures are better for single-file distribution. Detached `.mf.sig` files follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are simpler for HTTP serving. Support both modes?
duplicate scanner/checker implementations; export the manifest
type; canonical Go module path for 1.0 > *answer:*
- Format and correctness:
- Resolve proto go_package vs go.mod module path inconsistency **10. GPG vs pure-Go crypto for signatures?**
- Specify and validate path invariants (UTF-8, forward-slash, Shelling out to `gpg` is fragile (may not be installed, version-dependent output). `github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could go Ed25519/signify (simpler, no key management). Which direction?
relative, no .., no leading /)
- Remove or deprecate atime; reserve mode field; add version byte > *answer:*
(all pending design answers)
- Write a standalone format specification document ### Implementation Design
- Library:
- Delete internal/scanner and internal/checker; consolidate on the **11. Should manifests be deterministic by default?**
mfer/ package versions (pending design answer) This means: sort file entries by path, omit `createdAt` timestamp (or make it opt-in), no `atime`. Should determinism be the default, with a `--include-timestamps` flag to opt in?
- Add decompression size limit via io.LimitReader in
deserializeInner() > *answer:*
- Fix errors.Is dead code in checker; make AddFile verify
totalRead == size **12. Should we consolidate or keep both scanner/checker implementations?**
- Export manifest type or define a public interface (pending) There are two parallel implementations: `mfer/scanner.go` + `mfer/checker.go` (typed with `FileSize`, `RelFilePath`) and `internal/scanner/` + `internal/checker/` (raw `int64`, `string`). The `mfer/` versions are superior. Delete the `internal/` versions?
- Replace GPG subprocess with pure-Go crypto (pending); add timeouts
to remaining subprocess calls > *answer:*
- CLI:
- Kebab-case primary flag names; fix fetch URL construction with **13. Should the `manifest` type be exported?**
url.JoinPath; add http.Client timeout and retry with backoff to Currently unexported with exported constructors (`New`, `NewFromPaths`, etc.). Consumers can't declare `var m *mfer.manifest`. Export the type, or define an interface?
fetch; rate-limit Checker progress output; add --deterministic
flag or default; wire top-level --version properly > *answer:*
- Testing:
- Fuzz NewManifestFromReader; end-to-end tests for freshen and fetch **14. What should the Go module path be for 1.0?**
- Documentation: Currently mixed between `sneak.berlin/go/mfer` and `git.eeqj.de/sneak/mfer`. Which is canonical?
- Promote docs/FORMAT.md as primary spec reference; audit error
messages; document the signature scheme fully > *answer:*
- Release:
- Finalize module path, bump version constant, SemVer --version ---
output, tag v1.0.0
## Implementation Plan
### Phase 1: Foundation (format correctness)
- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate on `mfer/` package versions; update CLI code
- [ ] Add deterministic file ordering — sort entries by path (lexicographic, byte-order) in `Builder.Build()`; add test asserting byte-identical output from two runs
- [ ] Add decompression size limit — `io.LimitReader` in `deserializeInner()` with `m.pbOuter.Size` as bound
- [ ] Fix `errors.Is` dead code in checker — replace with `os.IsNotExist(err)` or `errors.Is(err, fs.ErrNotExist)`
- [ ] Fix `AddFile` to verify size — check `totalRead == size` after reading, return error on mismatch
- [ ] Specify path invariants — add proto comments (UTF-8, forward-slash, relative, no `..`, no leading `/`); validate in `Builder.AddFile` and `Builder.AddFileWithHash`
### Phase 2: CLI polish
- [ ] Fix flag naming — all CLI flags use kebab-case as primary (`--include-dotfiles`, `--follow-symlinks`)
- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or `url.JoinPath()` instead of string concatenation
- [ ] Add progress rate-limiting to Checker — throttle to once per second, matching Scanner
- [ ] Add `--deterministic` flag (or make it default) — omit `createdAt`, sort files
### Phase 3: Robustness
- [ ] Replace GPG subprocess with pure-Go crypto — `github.com/ProtonMail/go-crypto` or Ed25519/signify
- [ ] Add timeout to any remaining subprocess calls
- [ ] Add fuzzing tests for `NewManifestFromReader`
- [ ] Add retry logic to fetch — exponential backoff for transient HTTP errors
### Phase 4: Format finalization
- [ ] Remove or deprecate `atime` from proto (pending design question answer)
- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future file permissions
- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version 1
- [ ] Write format specification document — separate from README: magic, outer structure, compression, inner structure, path invariants, signature scheme, canonical serialization
### Phase 5: Release prep
- [ ] Finalize Go module path
- [ ] Audit all error messages for consistency and helpfulness
- [ ] Add `--version` output matching SemVer
- [ ] Tag v1.0.0

View File

@@ -3,7 +3,6 @@ package cli
import ( import (
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"io"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
@@ -35,32 +34,29 @@ func findManifest(fs afero.Fs, dir string) (string, error) {
func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error { func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
log.Debug("checkManifestOperation()") log.Debug("checkManifestOperation()")
manifestPath, err := mfa.resolveManifestArg(ctx) var manifestPath string
if err != nil { var err error
return fmt.Errorf("check: %w", err)
}
// URL manifests need to be downloaded to a temp file for the checker if ctx.Args().Len() > 0 {
if isHTTPURL(manifestPath) { arg := ctx.Args().Get(0)
rc, fetchErr := mfa.openManifestReader(manifestPath) // Check if arg is a directory or a file
if fetchErr != nil { info, statErr := mfa.Fs.Stat(arg)
return fmt.Errorf("check: %w", fetchErr) if statErr == nil && info.IsDir() {
// It's a directory, look for manifest inside
manifestPath, err = findManifest(mfa.Fs, arg)
if err != nil {
return err
}
} else {
// Treat as a file path
manifestPath = arg
} }
tmpFile, tmpErr := afero.TempFile(mfa.Fs, "", "mfer-manifest-*.mf") } else {
if tmpErr != nil { // No argument, look in current directory
_ = rc.Close() manifestPath, err = findManifest(mfa.Fs, ".")
return fmt.Errorf("check: failed to create temp file: %w", tmpErr) if err != nil {
return err
} }
tmpPath := tmpFile.Name()
_, cpErr := io.Copy(tmpFile, rc)
_ = rc.Close()
_ = tmpFile.Close()
if cpErr != nil {
_ = mfa.Fs.Remove(tmpPath)
return fmt.Errorf("check: failed to download manifest: %w", cpErr)
}
defer func() { _ = mfa.Fs.Remove(tmpPath) }()
manifestPath = tmpPath
} }
basePath := ctx.String("base") basePath := ctx.String("base")

View File

@@ -1,72 +0,0 @@
package cli
import (
"encoding/hex"
"encoding/json"
"fmt"
"time"
"github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/mfer"
)
// ExportEntry represents a single file entry in the exported JSON output.
type ExportEntry struct {
Path string `json:"path"`
Size int64 `json:"size"`
Hashes []string `json:"hashes"`
Mtime *string `json:"mtime,omitempty"`
Ctime *string `json:"ctime,omitempty"`
}
func (mfa *CLIApp) exportManifestOperation(ctx *cli.Context) error {
pathOrURL, err := mfa.resolveManifestArg(ctx)
if err != nil {
return fmt.Errorf("export: %w", err)
}
rc, err := mfa.openManifestReader(pathOrURL)
if err != nil {
return fmt.Errorf("export: %w", err)
}
defer func() { _ = rc.Close() }()
manifest, err := mfer.NewManifestFromReader(rc)
if err != nil {
return fmt.Errorf("export: failed to parse manifest: %w", err)
}
files := manifest.Files()
entries := make([]ExportEntry, 0, len(files))
for _, f := range files {
entry := ExportEntry{
Path: f.Path,
Size: f.Size,
Hashes: make([]string, 0, len(f.Hashes)),
}
for _, h := range f.Hashes {
entry.Hashes = append(entry.Hashes, hex.EncodeToString(h.MultiHash))
}
if f.Mtime != nil {
t := time.Unix(f.Mtime.Seconds, int64(f.Mtime.Nanos)).UTC().Format(time.RFC3339Nano)
entry.Mtime = &t
}
if f.Ctime != nil {
t := time.Unix(f.Ctime.Seconds, int64(f.Ctime.Nanos)).UTC().Format(time.RFC3339Nano)
entry.Ctime = &t
}
entries = append(entries, entry)
}
enc := json.NewEncoder(mfa.Stdout)
enc.SetIndent("", " ")
if err := enc.Encode(entries); err != nil {
return fmt.Errorf("export: failed to encode JSON: %w", err)
}
return nil
}

View File

@@ -1,137 +0,0 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/mfer/mfer"
)
// buildTestManifest creates a manifest from in-memory files and returns its bytes.
func buildTestManifest(t *testing.T, files map[string][]byte) []byte {
t.Helper()
sourceFs := afero.NewMemMapFs()
for path, content := range files {
require.NoError(t, sourceFs.MkdirAll("/", 0o755))
require.NoError(t, afero.WriteFile(sourceFs, "/"+path, content, 0o644))
}
opts := &mfer.ScannerOptions{Fs: sourceFs}
s := mfer.NewScannerWithOptions(opts)
require.NoError(t, s.EnumerateFS(sourceFs, "/", nil))
var buf bytes.Buffer
require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
return buf.Bytes()
}
func TestExportManifestOperation(t *testing.T) {
testFiles := map[string][]byte{
"hello.txt": []byte("Hello, World!"),
"sub/file.txt": []byte("nested content"),
}
manifestData := buildTestManifest(t, testFiles)
// Write manifest to memfs
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/test.mf", manifestData, 0o644))
var stdout, stderr bytes.Buffer
exitCode := RunWithOptions(&RunOptions{
Appname: "mfer",
Args: []string{"mfer", "export", "/test.mf"},
Stdin: &bytes.Buffer{},
Stdout: &stdout,
Stderr: &stderr,
Fs: fs,
})
require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
var entries []ExportEntry
require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries))
assert.Len(t, entries, 2)
// Verify entries have expected fields
pathSet := make(map[string]bool)
for _, e := range entries {
pathSet[e.Path] = true
assert.NotEmpty(t, e.Hashes, "entry %s should have hashes", e.Path)
assert.Greater(t, e.Size, int64(0), "entry %s should have positive size", e.Path)
}
assert.True(t, pathSet["hello.txt"])
assert.True(t, pathSet["sub/file.txt"])
}
func TestExportFromHTTPURL(t *testing.T) {
testFiles := map[string][]byte{
"a.txt": []byte("aaa"),
}
manifestData := buildTestManifest(t, testFiles)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write(manifestData)
}))
defer server.Close()
var stdout, stderr bytes.Buffer
exitCode := RunWithOptions(&RunOptions{
Appname: "mfer",
Args: []string{"mfer", "export", server.URL + "/index.mf"},
Stdin: &bytes.Buffer{},
Stdout: &stdout,
Stderr: &stderr,
Fs: afero.NewMemMapFs(),
})
require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
var entries []ExportEntry
require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries))
assert.Len(t, entries, 1)
assert.Equal(t, "a.txt", entries[0].Path)
}
func TestListFromHTTPURL(t *testing.T) {
testFiles := map[string][]byte{
"one.txt": []byte("1"),
"two.txt": []byte("22"),
}
manifestData := buildTestManifest(t, testFiles)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write(manifestData)
}))
defer server.Close()
var stdout, stderr bytes.Buffer
exitCode := RunWithOptions(&RunOptions{
Appname: "mfer",
Args: []string{"mfer", "list", server.URL + "/index.mf"},
Stdin: &bytes.Buffer{},
Stdout: &stdout,
Stderr: &stderr,
Fs: afero.NewMemMapFs(),
})
require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
output := stdout.String()
assert.Contains(t, output, "one.txt")
assert.Contains(t, output, "two.txt")
}
func TestIsHTTPURL(t *testing.T) {
assert.True(t, isHTTPURL("http://example.com/manifest.mf"))
assert.True(t, isHTTPURL("https://example.com/manifest.mf"))
assert.False(t, isHTTPURL("/local/path.mf"))
assert.False(t, isHTTPURL("relative/path.mf"))
assert.False(t, isHTTPURL("ftp://example.com/file"))
}

View File

@@ -67,7 +67,7 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
// Compute base URL (directory containing manifest) // Compute base URL (directory containing manifest)
baseURL, err := url.Parse(manifestURL) baseURL, err := url.Parse(manifestURL)
if err != nil { if err != nil {
return fmt.Errorf("fetch: invalid manifest URL: %w", err) return err
} }
baseURL.Path = path.Dir(baseURL.Path) baseURL.Path = path.Dir(baseURL.Path)
if !strings.HasSuffix(baseURL.Path, "/") { if !strings.HasSuffix(baseURL.Path, "/") {
@@ -113,7 +113,7 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
return fmt.Errorf("invalid path in manifest: %w", err) return fmt.Errorf("invalid path in manifest: %w", err)
} }
fileURL := baseURL.String() + encodeFilePath(f.Path) fileURL := baseURL.String() + f.Path
log.Infof("fetching %s", f.Path) log.Infof("fetching %s", f.Path)
if err := downloadFile(fileURL, localPath, f, progress); err != nil { if err := downloadFile(fileURL, localPath, f, progress); err != nil {
@@ -139,15 +139,6 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
return nil return nil
} }
// encodeFilePath URL-encodes each segment of a file path while preserving slashes.
func encodeFilePath(p string) string {
segments := strings.Split(p, "/")
for i, seg := range segments {
segments[i] = url.PathEscape(seg)
}
return strings.Join(segments, "/")
}
// sanitizePath validates and sanitizes a file path from the manifest. // sanitizePath validates and sanitizes a file path from the manifest.
// It prevents path traversal attacks and rejects unsafe paths. // It prevents path traversal attacks and rejects unsafe paths.
func sanitizePath(p string) (string, error) { func sanitizePath(p string) (string, error) {
@@ -267,7 +258,7 @@ func downloadFile(fileURL, localPath string, entry *mfer.MFFilePath, progress ch
dir := filepath.Dir(localPath) dir := filepath.Dir(localPath)
if dir != "" && dir != "." { if dir != "" && dir != "." {
if err := os.MkdirAll(dir, 0o755); err != nil { if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("failed to create directory %s: %w", dir, err) return err
} }
} }
@@ -287,9 +278,9 @@ func downloadFile(fileURL, localPath string, entry *mfer.MFFilePath, progress ch
} }
// Fetch file // Fetch file
resp, err := http.Get(fileURL) //nolint:gosec // URL constructed from manifest base resp, err := http.Get(fileURL)
if err != nil { if err != nil {
return fmt.Errorf("HTTP request failed: %w", err) return err
} }
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
@@ -307,7 +298,7 @@ func downloadFile(fileURL, localPath string, entry *mfer.MFFilePath, progress ch
// Create temp file // Create temp file
out, err := os.Create(tmpPath) out, err := os.Create(tmpPath)
if err != nil { if err != nil {
return fmt.Errorf("failed to create temp file: %w", err) return err
} }
// Set up hash computation // Set up hash computation

View File

@@ -16,29 +16,6 @@ import (
"sneak.berlin/go/mfer/mfer" "sneak.berlin/go/mfer/mfer"
) )
func TestEncodeFilePath(t *testing.T) {
tests := []struct {
input string
expected string
}{
{"file.txt", "file.txt"},
{"dir/file.txt", "dir/file.txt"},
{"my file.txt", "my%20file.txt"},
{"dir/my file.txt", "dir/my%20file.txt"},
{"file#1.txt", "file%231.txt"},
{"file?v=1.txt", "file%3Fv=1.txt"},
{"path/to/file with spaces.txt", "path/to/file%20with%20spaces.txt"},
{"100%done.txt", "100%25done.txt"},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
result := encodeFilePath(tt.input)
assert.Equal(t, tt.expected, result)
})
}
}
func TestSanitizePath(t *testing.T) { func TestSanitizePath(t *testing.T) {
// Valid paths that should be accepted // Valid paths that should be accepted
validTests := []struct { validTests := []struct {

View File

@@ -41,8 +41,8 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
basePath := ctx.String("base") basePath := ctx.String("base")
showProgress := ctx.Bool("progress") showProgress := ctx.Bool("progress")
includeDotfiles := ctx.Bool("include-dotfiles") includeDotfiles := ctx.Bool("IncludeDotfiles")
followSymlinks := ctx.Bool("follow-symlinks") followSymlinks := ctx.Bool("FollowSymLinks")
// Find manifest file // Find manifest file
var manifestPath string var manifestPath string
@@ -54,7 +54,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
if statErr == nil && info.IsDir() { if statErr == nil && info.IsDir() {
manifestPath, err = findManifest(mfa.Fs, arg) manifestPath, err = findManifest(mfa.Fs, arg)
if err != nil { if err != nil {
return fmt.Errorf("freshen: %w", err) return err
} }
} else { } else {
manifestPath = arg manifestPath = arg
@@ -62,7 +62,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
} else { } else {
manifestPath, err = findManifest(mfa.Fs, ".") manifestPath, err = findManifest(mfa.Fs, ".")
if err != nil { if err != nil {
return fmt.Errorf("freshen: %w", err) return err
} }
} }
@@ -93,7 +93,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
absBase, err := filepath.Abs(basePath) absBase, err := filepath.Abs(basePath)
if err != nil { if err != nil {
return fmt.Errorf("freshen: invalid base path: %w", err) return err
} }
err = afero.Walk(mfa.Fs, absBase, func(path string, info fs.FileInfo, walkErr error) error { err = afero.Walk(mfa.Fs, absBase, func(path string, info fs.FileInfo, walkErr error) error {
@@ -104,7 +104,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
// Get relative path // Get relative path
relPath, err := filepath.Rel(absBase, path) relPath, err := filepath.Rel(absBase, path)
if err != nil { if err != nil {
return fmt.Errorf("freshen: failed to compute relative path for %s: %w", path, err) return err
} }
// Skip the manifest file itself // Skip the manifest file itself
@@ -226,9 +226,6 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
var hashedBytes int64 var hashedBytes int64
builder := mfer.NewBuilder() builder := mfer.NewBuilder()
if ctx.Bool("include-timestamps") {
builder.SetIncludeTimestamps(true)
}
// Set up signing options if sign-key is provided // Set up signing options if sign-key is provided
if signKey := ctx.String("sign-key"); signKey != "" { if signKey := ctx.String("sign-key"); signKey != "" {

View File

@@ -20,16 +20,9 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
log.Debug("generateManifestOperation()") log.Debug("generateManifestOperation()")
opts := &mfer.ScannerOptions{ opts := &mfer.ScannerOptions{
IncludeDotfiles: ctx.Bool("include-dotfiles"), IncludeDotfiles: ctx.Bool("IncludeDotfiles"),
FollowSymLinks: ctx.Bool("follow-symlinks"), FollowSymLinks: ctx.Bool("FollowSymLinks"),
IncludeTimestamps: ctx.Bool("include-timestamps"), Fs: mfa.Fs,
Fs: mfa.Fs,
}
// Set seed for deterministic UUID if provided
if seed := ctx.String("seed"); seed != "" {
opts.Seed = seed
log.Infof("using deterministic seed for manifest UUID")
} }
// Set up signing options if sign-key is provided // Set up signing options if sign-key is provided
@@ -66,7 +59,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
if args.Len() == 0 { if args.Len() == 0 {
// Default to current directory // Default to current directory
if err := s.EnumeratePath(".", enumProgress); err != nil { if err := s.EnumeratePath(".", enumProgress); err != nil {
return fmt.Errorf("generate: failed to enumerate current directory: %w", err) return err
} }
} else { } else {
// Collect and validate all paths first // Collect and validate all paths first
@@ -75,7 +68,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
inputPath := args.Get(i) inputPath := args.Get(i)
ap, err := filepath.Abs(inputPath) ap, err := filepath.Abs(inputPath)
if err != nil { if err != nil {
return fmt.Errorf("generate: invalid path %q: %w", inputPath, err) return err
} }
// Validate path exists before adding to list // Validate path exists before adding to list
if exists, _ := afero.Exists(mfa.Fs, ap); !exists { if exists, _ := afero.Exists(mfa.Fs, ap); !exists {
@@ -85,7 +78,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
paths = append(paths, ap) paths = append(paths, ap)
} }
if err := s.EnumeratePaths(enumProgress, paths...); err != nil { if err := s.EnumeratePaths(enumProgress, paths...); err != nil {
return fmt.Errorf("generate: failed to enumerate paths: %w", err) return err
} }
} }
enumWg.Wait() enumWg.Wait()

View File

@@ -16,20 +16,32 @@ func (mfa *CLIApp) listManifestOperation(ctx *cli.Context) error {
longFormat := ctx.Bool("long") longFormat := ctx.Bool("long")
print0 := ctx.Bool("print0") print0 := ctx.Bool("print0")
pathOrURL, err := mfa.resolveManifestArg(ctx) // Find manifest file
if err != nil { var manifestPath string
return fmt.Errorf("list: %w", err) var err error
if ctx.Args().Len() > 0 {
arg := ctx.Args().Get(0)
info, statErr := mfa.Fs.Stat(arg)
if statErr == nil && info.IsDir() {
manifestPath, err = findManifest(mfa.Fs, arg)
if err != nil {
return err
}
} else {
manifestPath = arg
}
} else {
manifestPath, err = findManifest(mfa.Fs, ".")
if err != nil {
return err
}
} }
rc, err := mfa.openManifestReader(pathOrURL) // Load manifest
manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath)
if err != nil { if err != nil {
return fmt.Errorf("list: %w", err) return fmt.Errorf("failed to load manifest: %w", err)
}
defer func() { _ = rc.Close() }()
manifest, err := mfer.NewManifestFromReader(rc)
if err != nil {
return fmt.Errorf("list: failed to parse manifest: %w", err)
} }
files := manifest.Files() files := manifest.Files()

View File

@@ -1,56 +0,0 @@
package cli
import (
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/urfave/cli/v2"
)
// isHTTPURL returns true if the string starts with http:// or https://.
func isHTTPURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
}
// openManifestReader opens a manifest from a path or URL and returns a ReadCloser.
// The caller must close the returned reader.
func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
if isHTTPURL(pathOrURL) {
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Get(pathOrURL) //nolint:gosec // user-provided URL is intentional
if err != nil {
return nil, fmt.Errorf("failed to fetch %s: %w", pathOrURL, err)
}
if resp.StatusCode != http.StatusOK {
_ = resp.Body.Close()
return nil, fmt.Errorf("failed to fetch %s: HTTP %d", pathOrURL, resp.StatusCode)
}
return resp.Body, nil
}
f, err := mfa.Fs.Open(pathOrURL)
if err != nil {
return nil, err
}
return f, nil
}
// resolveManifestArg resolves the manifest path from CLI arguments.
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf.
// If no argument is given, the current directory is searched.
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
if ctx.Args().Len() > 0 {
arg := ctx.Args().Get(0)
if isHTTPURL(arg) {
return arg, nil
}
info, statErr := mfa.Fs.Stat(arg)
if statErr == nil && info.IsDir() {
return findManifest(mfa.Fs, arg)
}
return arg, nil
}
return findManifest(mfa.Fs, ".")
}

View File

@@ -123,15 +123,14 @@ func (mfa *CLIApp) run(args []string) {
}, },
Flags: append(commonFlags(), Flags: append(commonFlags(),
&cli.BoolFlag{ &cli.BoolFlag{
Name: "follow-symlinks", Name: "FollowSymLinks",
Aliases: []string{"L"}, Aliases: []string{"follow-symlinks"},
Usage: "Resolve encountered symlinks", Usage: "Resolve encountered symlinks",
}, },
&cli.BoolFlag{ &cli.BoolFlag{
Name: "include-dotfiles", Name: "IncludeDotfiles",
Aliases: []string{"IncludeDotfiles"}, Aliases: []string{"include-dotfiles"},
Usage: "Include dot (hidden) files (excluded by default)",
Usage: "Include dot (hidden) files (excluded by default)",
}, },
&cli.StringFlag{ &cli.StringFlag{
Name: "output", Name: "output",
@@ -155,15 +154,6 @@ func (mfa *CLIApp) run(args []string) {
Usage: "GPG key ID to sign the manifest with", Usage: "GPG key ID to sign the manifest with",
EnvVars: []string{"MFER_SIGN_KEY"}, EnvVars: []string{"MFER_SIGN_KEY"},
}, },
&cli.StringFlag{
Name: "seed",
Usage: "Seed value for deterministic manifest UUID",
EnvVars: []string{"MFER_SEED"},
},
&cli.BoolFlag{
Name: "include-timestamps",
Usage: "Include createdAt timestamp in manifest (omitted by default for determinism)",
},
), ),
}, },
{ {
@@ -216,15 +206,14 @@ func (mfa *CLIApp) run(args []string) {
Usage: "Base directory for resolving relative paths", Usage: "Base directory for resolving relative paths",
}, },
&cli.BoolFlag{ &cli.BoolFlag{
Name: "follow-symlinks", Name: "FollowSymLinks",
Aliases: []string{"L"}, Aliases: []string{"follow-symlinks"},
Usage: "Resolve encountered symlinks", Usage: "Resolve encountered symlinks",
}, },
&cli.BoolFlag{ &cli.BoolFlag{
Name: "include-dotfiles", Name: "IncludeDotfiles",
Aliases: []string{"IncludeDotfiles"}, Aliases: []string{"include-dotfiles"},
Usage: "Include dot (hidden) files (excluded by default)",
Usage: "Include dot (hidden) files (excluded by default)",
}, },
&cli.BoolFlag{ &cli.BoolFlag{
Name: "progress", Name: "progress",
@@ -237,20 +226,8 @@ func (mfa *CLIApp) run(args []string) {
Usage: "GPG key ID to sign the manifest with", Usage: "GPG key ID to sign the manifest with",
EnvVars: []string{"MFER_SIGN_KEY"}, EnvVars: []string{"MFER_SIGN_KEY"},
}, },
&cli.BoolFlag{
Name: "include-timestamps",
Usage: "Include createdAt timestamp in manifest (omitted by default for determinism)",
},
), ),
}, },
{
Name: "export",
Usage: "Export manifest contents as JSON",
ArgsUsage: "[manifest file or URL]",
Action: func(c *cli.Context) error {
return mfa.exportManifestOperation(c)
},
},
{ {
Name: "version", Name: "version",
Usage: "Show version", Usage: "Show version",
@@ -292,7 +269,7 @@ func (mfa *CLIApp) run(args []string) {
}, },
} }
mfa.app.HideVersion = false mfa.app.HideVersion = true
err := mfa.app.Run(args) err := mfa.app.Run(args)
if err != nil { if err != nil {
mfa.exitCode = 1 mfa.exitCode = 1

View File

@@ -5,7 +5,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"sort"
"strings" "strings"
"sync" "sync"
"time" "time"
@@ -85,20 +84,10 @@ type FileHashProgress struct {
// Builder constructs a manifest by adding files one at a time. // Builder constructs a manifest by adding files one at a time.
type Builder struct { type Builder struct {
mu sync.Mutex mu sync.Mutex
files []*MFFilePath files []*MFFilePath
createdAt time.Time createdAt time.Time
includeTimestamps bool signingOptions *SigningOptions
signingOptions *SigningOptions
fixedUUID []byte // if set, use this UUID instead of generating one
}
// SetSeed derives a deterministic UUID from the given seed string.
// The seed is hashed once with SHA-256 and the first 16 bytes are used
// as a fixed UUID for the manifest.
func (b *Builder) SetSeed(seed string) {
hash := sha256.Sum256([]byte(seed))
b.fixedUUID = hash[:16]
} }
// NewBuilder creates a new Builder. // NewBuilder creates a new Builder.
@@ -196,7 +185,7 @@ func (b *Builder) FileCount() int {
// Returns an error if path is empty, size is negative, or hash is nil/empty. // Returns an error if path is empty, size is negative, or hash is nil/empty.
func (b *Builder) AddFileWithHash(path RelFilePath, size FileSize, mtime ModTime, hash Multihash) error { func (b *Builder) AddFileWithHash(path RelFilePath, size FileSize, mtime ModTime, hash Multihash) error {
if err := ValidatePath(string(path)); err != nil { if err := ValidatePath(string(path)); err != nil {
return fmt.Errorf("add file: %w", err) return err
} }
if size < 0 { if size < 0 {
return errors.New("size cannot be negative") return errors.New("size cannot be negative")
@@ -220,14 +209,6 @@ func (b *Builder) AddFileWithHash(path RelFilePath, size FileSize, mtime ModTime
return nil return nil
} }
// SetIncludeTimestamps controls whether the manifest includes a createdAt timestamp.
// By default timestamps are omitted for deterministic output.
func (b *Builder) SetIncludeTimestamps(include bool) {
b.mu.Lock()
defer b.mu.Unlock()
b.includeTimestamps = include
}
// SetSigningOptions sets the GPG signing options for the manifest. // SetSigningOptions sets the GPG signing options for the manifest.
// If opts is non-nil, the manifest will be signed when Build() is called. // If opts is non-nil, the manifest will be signed when Build() is called.
func (b *Builder) SetSigningOptions(opts *SigningOptions) { func (b *Builder) SetSigningOptions(opts *SigningOptions) {
@@ -241,41 +222,30 @@ func (b *Builder) Build(w io.Writer) error {
b.mu.Lock() b.mu.Lock()
defer b.mu.Unlock() defer b.mu.Unlock()
// Sort files by path for deterministic output
sort.Slice(b.files, func(i, j int) bool {
return b.files[i].Path < b.files[j].Path
})
// Create inner manifest // Create inner manifest
inner := &MFFile{ inner := &MFFile{
Version: MFFile_VERSION_ONE, Version: MFFile_VERSION_ONE,
Files: b.files, CreatedAt: newTimestampFromTime(b.createdAt),
} Files: b.files,
if b.includeTimestamps {
inner.CreatedAt = newTimestampFromTime(b.createdAt)
} }
// Create a temporary manifest to use existing serialization // Create a temporary manifest to use existing serialization
m := &manifest{ m := &manifest{
pbInner: inner, pbInner: inner,
signingOptions: b.signingOptions, signingOptions: b.signingOptions,
fixedUUID: b.fixedUUID,
} }
// Generate outer wrapper // Generate outer wrapper
if err := m.generateOuter(); err != nil { if err := m.generateOuter(); err != nil {
return fmt.Errorf("build: generate outer: %w", err) return err
} }
// Generate final output // Generate final output
if err := m.generate(); err != nil { if err := m.generate(); err != nil {
return fmt.Errorf("build: generate: %w", err) return err
} }
// Write to output // Write to output
_, err := w.Write(m.output.Bytes()) _, err := w.Write(m.output.Bytes())
if err != nil { return err
return fmt.Errorf("build: write output: %w", err)
}
return nil
} }

View File

@@ -115,207 +115,6 @@ func TestNewTimestampFromTimeExtremeDate(t *testing.T) {
} }
} }
func TestBuilderDeterministicOutput(t *testing.T) {
buildManifest := func() []byte {
b := NewBuilder()
// Use a fixed createdAt and UUID so output is reproducible
b.createdAt = time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
b.fixedUUID = make([]byte, 16) // all zeros
mtime := ModTime(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC))
// Add files in reverse order to test sorting
files := []struct {
path string
content string
}{
{"c/file.txt", "content c"},
{"a/file.txt", "content a"},
{"b/file.txt", "content b"},
}
for _, f := range files {
r := bytes.NewReader([]byte(f.content))
_, err := b.AddFile(RelFilePath(f.path), FileSize(len(f.content)), mtime, r, nil)
require.NoError(t, err)
}
var buf bytes.Buffer
err := b.Build(&buf)
require.NoError(t, err)
return buf.Bytes()
}
out1 := buildManifest()
out2 := buildManifest()
assert.Equal(t, out1, out2, "two builds with same input should produce byte-identical output")
}
func TestSetSeedDeterministic(t *testing.T) {
b1 := NewBuilder()
b1.SetSeed("test-seed-value")
b2 := NewBuilder()
b2.SetSeed("test-seed-value")
assert.Equal(t, b1.fixedUUID, b2.fixedUUID, "same seed should produce same UUID")
assert.Len(t, b1.fixedUUID, 16, "UUID should be 16 bytes")
b3 := NewBuilder()
b3.SetSeed("different-seed")
assert.NotEqual(t, b1.fixedUUID, b3.fixedUUID, "different seeds should produce different UUIDs")
}
func TestValidatePath(t *testing.T) {
valid := []string{
"file.txt",
"dir/file.txt",
"a/b/c/d.txt",
"file with spaces.txt",
"日本語.txt",
}
for _, p := range valid {
t.Run("valid:"+p, func(t *testing.T) {
assert.NoError(t, ValidatePath(p))
})
}
invalid := []struct {
path string
desc string
}{
{"", "empty"},
{"/absolute", "absolute path"},
{"has\\backslash", "backslash"},
{"has/../traversal", "dot-dot segment"},
{"has//double", "empty segment"},
{"..", "just dot-dot"},
{string([]byte{0xff, 0xfe}), "invalid UTF-8"},
}
for _, tt := range invalid {
t.Run("invalid:"+tt.desc, func(t *testing.T) {
assert.Error(t, ValidatePath(tt.path))
})
}
}
func TestBuilderAddFileSizeMismatch(t *testing.T) {
b := NewBuilder()
content := []byte("short")
reader := bytes.NewReader(content)
// Declare wrong size
_, err := b.AddFile("test.txt", FileSize(100), ModTime(time.Now()), reader, nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "size mismatch")
}
func TestBuilderAddFileInvalidPath(t *testing.T) {
b := NewBuilder()
content := []byte("data")
reader := bytes.NewReader(content)
_, err := b.AddFile("", FileSize(len(content)), ModTime(time.Now()), reader, nil)
assert.Error(t, err)
reader.Reset(content)
_, err = b.AddFile("/absolute", FileSize(len(content)), ModTime(time.Now()), reader, nil)
assert.Error(t, err)
}
func TestBuilderAddFileWithProgress(t *testing.T) {
b := NewBuilder()
content := bytes.Repeat([]byte("x"), 1000)
reader := bytes.NewReader(content)
progress := make(chan FileHashProgress, 100)
bytesRead, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), reader, progress)
close(progress)
require.NoError(t, err)
assert.Equal(t, FileSize(1000), bytesRead)
var updates []FileHashProgress
for p := range progress {
updates = append(updates, p)
}
assert.NotEmpty(t, updates)
// Last update should show all bytes
assert.Equal(t, FileSize(1000), updates[len(updates)-1].BytesRead)
}
func TestBuilderBuildRoundTrip(t *testing.T) {
// Build a manifest, deserialize it, verify all fields survive round-trip
b := NewBuilder()
now := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)
files := []struct {
path string
content []byte
}{
{"alpha.txt", []byte("alpha content")},
{"beta/gamma.txt", []byte("gamma content")},
{"beta/delta.txt", []byte("delta content")},
}
for _, f := range files {
reader := bytes.NewReader(f.content)
_, err := b.AddFile(RelFilePath(f.path), FileSize(len(f.content)), ModTime(now), reader, nil)
require.NoError(t, err)
}
var buf bytes.Buffer
require.NoError(t, b.Build(&buf))
m, err := NewManifestFromReader(&buf)
require.NoError(t, err)
mfiles := m.Files()
require.Len(t, mfiles, 3)
// Verify sorted order
assert.Equal(t, "alpha.txt", mfiles[0].Path)
assert.Equal(t, "beta/delta.txt", mfiles[1].Path)
assert.Equal(t, "beta/gamma.txt", mfiles[2].Path)
// Verify sizes
assert.Equal(t, int64(len("alpha content")), mfiles[0].Size)
// Verify hashes are present
for _, f := range mfiles {
require.NotEmpty(t, f.Hashes, "file %s should have hashes", f.Path)
assert.NotEmpty(t, f.Hashes[0].MultiHash)
}
}
func TestNewManifestFromReaderInvalidMagic(t *testing.T) {
_, err := NewManifestFromReader(bytes.NewReader([]byte("NOT_VALID")))
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid file format")
}
func TestNewManifestFromReaderEmpty(t *testing.T) {
_, err := NewManifestFromReader(bytes.NewReader([]byte{}))
assert.Error(t, err)
}
func TestNewManifestFromReaderTruncated(t *testing.T) {
// Just the magic with nothing after
_, err := NewManifestFromReader(bytes.NewReader([]byte(MAGIC)))
assert.Error(t, err)
}
func TestManifestString(t *testing.T) {
b := NewBuilder()
content := []byte("test")
reader := bytes.NewReader(content)
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil)
require.NoError(t, err)
var buf bytes.Buffer
require.NoError(t, b.Build(&buf))
m, err := NewManifestFromReader(&buf)
require.NoError(t, err)
assert.Contains(t, m.String(), "count=1")
}
func TestBuilderBuildEmpty(t *testing.T) { func TestBuilderBuildEmpty(t *testing.T) {
b := NewBuilder() b := NewBuilder()
@@ -326,62 +125,3 @@ func TestBuilderBuildEmpty(t *testing.T) {
// Should still produce valid manifest with 0 files // Should still produce valid manifest with 0 files
assert.True(t, strings.HasPrefix(buf.String(), MAGIC)) assert.True(t, strings.HasPrefix(buf.String(), MAGIC))
} }
func TestBuilderOmitsCreatedAtByDefault(t *testing.T) {
b := NewBuilder()
content := []byte("hello")
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), bytes.NewReader(content), nil)
require.NoError(t, err)
var buf bytes.Buffer
require.NoError(t, b.Build(&buf))
m, err := NewManifestFromReader(&buf)
require.NoError(t, err)
assert.Nil(t, m.pbInner.CreatedAt, "createdAt should be nil by default for deterministic output")
}
func TestBuilderIncludesCreatedAtWhenRequested(t *testing.T) {
b := NewBuilder()
b.SetIncludeTimestamps(true)
content := []byte("hello")
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), bytes.NewReader(content), nil)
require.NoError(t, err)
var buf bytes.Buffer
require.NoError(t, b.Build(&buf))
m, err := NewManifestFromReader(&buf)
require.NoError(t, err)
assert.NotNil(t, m.pbInner.CreatedAt, "createdAt should be set when IncludeTimestamps is true")
}
func TestBuilderDeterministicFileOrder(t *testing.T) {
// Two builds with same files in different order should produce same file ordering.
// Note: UUIDs differ per build, so we compare parsed file lists, not raw bytes.
buildAndParse := func(order []string) []*MFFilePath {
b := NewBuilder()
for _, name := range order {
content := []byte("content of " + name)
_, err := b.AddFile(RelFilePath(name), FileSize(len(content)), ModTime(time.Unix(1000, 0)), bytes.NewReader(content), nil)
require.NoError(t, err)
}
var buf bytes.Buffer
require.NoError(t, b.Build(&buf))
m, err := NewManifestFromReader(&buf)
require.NoError(t, err)
return m.Files()
}
files1 := buildAndParse([]string{"b.txt", "a.txt"})
files2 := buildAndParse([]string{"a.txt", "b.txt"})
require.Len(t, files1, 2)
require.Len(t, files2, 2)
for i := range files1 {
assert.Equal(t, files1[i].Path, files2[i].Path)
assert.Equal(t, files1[i].Size, files2[i].Size)
}
assert.Equal(t, "a.txt", files1[0].Path)
assert.Equal(t, "b.txt", files1[1].Path)
}

View File

@@ -70,8 +70,6 @@ type Checker struct {
fs afero.Fs fs afero.Fs
// manifestPaths is a set of paths in the manifest for quick lookup // manifestPaths is a set of paths in the manifest for quick lookup
manifestPaths map[RelFilePath]struct{} manifestPaths map[RelFilePath]struct{}
// manifestRelPath is the relative path of the manifest file from basePath (for exclusion)
manifestRelPath RelFilePath
// signature info from the manifest // signature info from the manifest
signature []byte signature []byte
signer []byte signer []byte
@@ -102,25 +100,14 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
manifestPaths[RelFilePath(f.Path)] = struct{}{} manifestPaths[RelFilePath(f.Path)] = struct{}{}
} }
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles
absManifest, err := filepath.Abs(manifestPath)
if err != nil {
return nil, err
}
manifestRel, err := filepath.Rel(abs, absManifest)
if err != nil {
manifestRel = ""
}
return &Checker{ return &Checker{
basePath: AbsFilePath(abs), basePath: AbsFilePath(abs),
files: files, files: files,
fs: fs, fs: fs,
manifestPaths: manifestPaths, manifestPaths: manifestPaths,
manifestRelPath: RelFilePath(manifestRel), signature: m.pbOuter.Signature,
signature: m.pbOuter.Signature, signer: m.pbOuter.Signer,
signer: m.pbOuter.Signer, signingPubKey: m.pbOuter.SigningPubKey,
signingPubKey: m.pbOuter.SigningPubKey,
}, nil }, nil
} }
@@ -183,7 +170,6 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
var failures FileCount var failures FileCount
startTime := time.Now() startTime := time.Now()
lastProgressTime := time.Now()
for _, entry := range c.files { for _, entry := range c.files {
select { select {
@@ -202,34 +188,29 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
results <- result results <- result
} }
// Send progress at most once per second (rate-limited) // Send progress with rate and ETA calculation
if progress != nil { if progress != nil {
now := time.Now() elapsed := time.Since(startTime)
isLast := checkedFiles == totalFiles var bytesPerSec float64
if isLast || now.Sub(lastProgressTime) >= time.Second { var eta time.Duration
elapsed := time.Since(startTime)
var bytesPerSec float64
var eta time.Duration
if elapsed > 0 && checkedBytes > 0 { if elapsed > 0 && checkedBytes > 0 {
bytesPerSec = float64(checkedBytes) / elapsed.Seconds() bytesPerSec = float64(checkedBytes) / elapsed.Seconds()
remainingBytes := totalBytes - checkedBytes remainingBytes := totalBytes - checkedBytes
if bytesPerSec > 0 { if bytesPerSec > 0 {
eta = time.Duration(float64(remainingBytes)/bytesPerSec) * time.Second eta = time.Duration(float64(remainingBytes)/bytesPerSec) * time.Second
}
} }
sendCheckStatus(progress, CheckStatus{
TotalFiles: totalFiles,
CheckedFiles: checkedFiles,
TotalBytes: totalBytes,
CheckedBytes: checkedBytes,
BytesPerSec: bytesPerSec,
ETA: eta,
Failures: failures,
})
lastProgressTime = now
} }
sendCheckStatus(progress, CheckStatus{
TotalFiles: totalFiles,
CheckedFiles: checkedFiles,
TotalBytes: totalBytes,
CheckedBytes: checkedBytes,
BytesPerSec: bytesPerSec,
ETA: eta,
Failures: failures,
})
} }
} }
@@ -291,14 +272,12 @@ func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
// FindExtraFiles walks the filesystem and reports files not in the manifest. // FindExtraFiles walks the filesystem and reports files not in the manifest.
// Results are sent to the results channel. The channel is closed when done. // Results are sent to the results channel. The channel is closed when done.
// Hidden files/directories (starting with .) are skipped, as they are excluded
// from manifests by default. The manifest file itself is also skipped.
func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error { func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error {
if results != nil { if results != nil {
defer close(results) defer close(results)
} }
return afero.Walk(c.fs, string(c.basePath), func(walkPath string, info os.FileInfo, err error) error { return afero.Walk(c.fs, string(c.basePath), func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return err return err
} }
@@ -309,31 +288,17 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
default: default:
} }
// Get relative path
rel, err := filepath.Rel(string(c.basePath), walkPath)
if err != nil {
return err
}
// Skip hidden files and directories (dotfiles)
if IsHiddenPath(filepath.ToSlash(rel)) {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
// Skip directories // Skip directories
if info.IsDir() { if info.IsDir() {
return nil return nil
} }
relPath := RelFilePath(rel) // Get relative path
rel, err := filepath.Rel(string(c.basePath), path)
// Skip the manifest file itself if err != nil {
if relPath == c.manifestRelPath { return err
return nil
} }
relPath := RelFilePath(rel)
// Check if path is in manifest // Check if path is in manifest
if _, exists := c.manifestPaths[relPath]; !exists { if _, exists := c.manifestPaths[relPath]; !exists {

View File

@@ -3,7 +3,6 @@ package mfer
import ( import (
"bytes" "bytes"
"context" "context"
"fmt"
"testing" "testing"
"time" "time"
@@ -306,44 +305,6 @@ func TestFindExtraFiles(t *testing.T) {
assert.Equal(t, "not in manifest", extras[0].Message) assert.Equal(t, "not in manifest", extras[0].Message)
} }
func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
fs := afero.NewMemMapFs()
manifestFiles := map[string][]byte{
"file1.txt": []byte("in manifest"),
}
createTestManifest(t, fs, "/data/.index.mf", manifestFiles)
createFilesOnDisk(t, fs, "/data", map[string][]byte{
"file1.txt": []byte("in manifest"),
})
// Create dotfile and manifest that should be skipped
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
// Create a real extra file
require.NoError(t, fs.MkdirAll("/data", 0o755))
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
chk, err := NewChecker("/data/.index.mf", "/data", fs)
require.NoError(t, err)
results := make(chan Result, 10)
err = chk.FindExtraFiles(context.Background(), results)
require.NoError(t, err)
var extras []Result
for r := range results {
extras = append(extras, r)
}
// Should only report extra.txt, not .hidden, .config/settings, or .index.mf
for _, e := range extras {
t.Logf("extra: %s", e.Path)
}
assert.Len(t, extras, 1)
if len(extras) > 0 {
assert.Equal(t, RelFilePath("extra.txt"), extras[0].Path)
}
}
func TestFindExtraFilesContextCancellation(t *testing.T) { func TestFindExtraFilesContextCancellation(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
files := map[string][]byte{"file.txt": []byte("data")} files := map[string][]byte{"file.txt": []byte("data")}
@@ -453,61 +414,6 @@ func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
assert.Equal(t, 0, statusCounts[StatusError], "no files should be ERROR") assert.Equal(t, 0, statusCounts[StatusError], "no files should be ERROR")
} }
func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
// Regression test for #16: FindExtraFiles should not report dotfiles
// or the manifest file itself as extra files.
fs := afero.NewMemMapFs()
files := map[string][]byte{
"file1.txt": []byte("in manifest"),
}
createTestManifest(t, fs, "/data/.index.mf", files)
createFilesOnDisk(t, fs, "/data", files)
// Add dotfiles and manifest file on disk
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("dotfile"), 0o644))
require.NoError(t, fs.MkdirAll("/data/.git", 0o755))
require.NoError(t, afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
chk, err := NewChecker("/data/.index.mf", "/data", fs)
require.NoError(t, err)
results := make(chan Result, 10)
err = chk.FindExtraFiles(context.Background(), results)
require.NoError(t, err)
var extras []Result
for r := range results {
extras = append(extras, r)
}
// Should report NO extra files — dotfiles and manifest should be skipped
assert.Empty(t, extras, "FindExtraFiles should not report dotfiles or manifest file as extra; got: %v", extras)
}
func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
// The manifest file itself should never be reported as extra
fs := afero.NewMemMapFs()
files := map[string][]byte{
"file1.txt": []byte("content"),
}
createTestManifest(t, fs, "/data/index.mf", files)
createFilesOnDisk(t, fs, "/data", files)
chk, err := NewChecker("/data/index.mf", "/data", fs)
require.NoError(t, err)
results := make(chan Result, 10)
err = chk.FindExtraFiles(context.Background(), results)
require.NoError(t, err)
var extras []Result
for r := range results {
extras = append(extras, r)
}
assert.Empty(t, extras, "manifest file should not be reported as extra; got: %v", extras)
}
func TestCheckEmptyManifest(t *testing.T) { func TestCheckEmptyManifest(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
// Create manifest with no files // Create manifest with no files
@@ -529,40 +435,3 @@ func TestCheckEmptyManifest(t *testing.T) {
} }
assert.Equal(t, 0, count) assert.Equal(t, 0, count)
} }
func TestCheckProgressRateLimited(t *testing.T) {
// Create many small files - progress should be rate-limited, not one per file.
// With rate-limiting to once per second, we should get far fewer progress
// updates than files (plus one final update).
fs := afero.NewMemMapFs()
files := make(map[string][]byte, 100)
for i := 0; i < 100; i++ {
name := fmt.Sprintf("file%03d.txt", i)
files[name] = []byte("content")
}
createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, "/data", files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err)
results := make(chan Result, 200)
progress := make(chan CheckStatus, 200)
err = chk.Check(context.Background(), results, progress)
require.NoError(t, err)
// Drain results
for range results {
}
// Count progress updates
var progressCount int
for range progress {
progressCount++
}
// Should be far fewer than 100 (rate-limited to once per second)
// At minimum we get the final update
assert.GreaterOrEqual(t, progressCount, 1, "should get at least the final progress update")
assert.Less(t, progressCount, 100, "progress should be rate-limited, not one per file")
}

View File

@@ -44,7 +44,7 @@ func (m *manifest) deserializeInner() error {
// Verify hash of compressed data before decompression // Verify hash of compressed data before decompression
h := sha256.New() h := sha256.New()
if _, err := h.Write(m.pbOuter.InnerMessage); err != nil { if _, err := h.Write(m.pbOuter.InnerMessage); err != nil {
return fmt.Errorf("deserialize: hash write: %w", err) return err
} }
sha256Hash := h.Sum(nil) sha256Hash := h.Sum(nil)
if !bytes.Equal(sha256Hash, m.pbOuter.Sha256) { if !bytes.Equal(sha256Hash, m.pbOuter.Sha256) {
@@ -72,7 +72,7 @@ func (m *manifest) deserializeInner() error {
zr, err := zstd.NewReader(bb) zr, err := zstd.NewReader(bb)
if err != nil { if err != nil {
return fmt.Errorf("deserialize: zstd reader: %w", err) return err
} }
defer zr.Close() defer zr.Close()
@@ -85,7 +85,7 @@ func (m *manifest) deserializeInner() error {
limitedReader := io.LimitReader(zr, maxSize) limitedReader := io.LimitReader(zr, maxSize)
dat, err := io.ReadAll(limitedReader) dat, err := io.ReadAll(limitedReader)
if err != nil { if err != nil {
return fmt.Errorf("deserialize: decompress: %w", err) return err
} }
if int64(len(dat)) >= MaxDecompressedSize { if int64(len(dat)) >= MaxDecompressedSize {
return fmt.Errorf("decompressed data exceeds maximum allowed size of %d bytes", MaxDecompressedSize) return fmt.Errorf("decompressed data exceeds maximum allowed size of %d bytes", MaxDecompressedSize)
@@ -100,7 +100,7 @@ func (m *manifest) deserializeInner() error {
// Deserialize inner message // Deserialize inner message
m.pbInner = new(MFFile) m.pbInner = new(MFFile)
if err := proto.Unmarshal(dat, m.pbInner); err != nil { if err := proto.Unmarshal(dat, m.pbInner); err != nil {
return fmt.Errorf("deserialize: unmarshal inner: %w", err) return err
} }
// Validate inner UUID // Validate inner UUID

View File

@@ -20,7 +20,7 @@ type SigningOptions struct {
// gpgSign creates a detached signature of the data using the specified key. // gpgSign creates a detached signature of the data using the specified key.
// Returns the armored detached signature. // Returns the armored detached signature.
func gpgSign(data []byte, keyID GPGKeyID) ([]byte, error) { func gpgSign(data []byte, keyID GPGKeyID) ([]byte, error) {
cmd := exec.Command("gpg", "--batch", "--no-tty", cmd := exec.Command("gpg",
"--detach-sign", "--detach-sign",
"--armor", "--armor",
"--local-user", string(keyID), "--local-user", string(keyID),
@@ -42,7 +42,7 @@ func gpgSign(data []byte, keyID GPGKeyID) ([]byte, error) {
// gpgExportPublicKey exports the public key for the specified key ID. // gpgExportPublicKey exports the public key for the specified key ID.
// Returns the armored public key. // Returns the armored public key.
func gpgExportPublicKey(keyID GPGKeyID) ([]byte, error) { func gpgExportPublicKey(keyID GPGKeyID) ([]byte, error) {
cmd := exec.Command("gpg", "--batch", "--no-tty", cmd := exec.Command("gpg",
"--export", "--export",
"--armor", "--armor",
string(keyID), string(keyID),
@@ -65,7 +65,7 @@ func gpgExportPublicKey(keyID GPGKeyID) ([]byte, error) {
// gpgGetKeyFingerprint gets the full fingerprint for a key ID. // gpgGetKeyFingerprint gets the full fingerprint for a key ID.
func gpgGetKeyFingerprint(keyID GPGKeyID) ([]byte, error) { func gpgGetKeyFingerprint(keyID GPGKeyID) ([]byte, error) {
cmd := exec.Command("gpg", "--batch", "--no-tty", cmd := exec.Command("gpg",
"--with-colons", "--with-colons",
"--fingerprint", "--fingerprint",
string(keyID), string(keyID),
@@ -100,7 +100,7 @@ func gpgExtractPubKeyFingerprint(pubKey []byte) (string, error) {
if err != nil { if err != nil {
return "", fmt.Errorf("failed to create temp dir: %w", err) return "", fmt.Errorf("failed to create temp dir: %w", err)
} }
defer func() { _ = os.RemoveAll(tmpDir) }() defer os.RemoveAll(tmpDir)
// Set restrictive permissions // Set restrictive permissions
if err := os.Chmod(tmpDir, 0o700); err != nil { if err := os.Chmod(tmpDir, 0o700); err != nil {
@@ -114,7 +114,7 @@ func gpgExtractPubKeyFingerprint(pubKey []byte) (string, error) {
} }
// Import the public key into the temporary keyring // Import the public key into the temporary keyring
importCmd := exec.Command("gpg", "--batch", "--no-tty", importCmd := exec.Command("gpg",
"--homedir", tmpDir, "--homedir", tmpDir,
"--import", "--import",
pubKeyFile, pubKeyFile,
@@ -126,7 +126,7 @@ func gpgExtractPubKeyFingerprint(pubKey []byte) (string, error) {
} }
// List keys to get fingerprint // List keys to get fingerprint
listCmd := exec.Command("gpg", "--batch", "--no-tty", listCmd := exec.Command("gpg",
"--homedir", tmpDir, "--homedir", tmpDir,
"--with-colons", "--with-colons",
"--fingerprint", "--fingerprint",
@@ -158,7 +158,7 @@ func gpgVerify(data, signature, pubKey []byte) error {
if err != nil { if err != nil {
return fmt.Errorf("failed to create temp dir: %w", err) return fmt.Errorf("failed to create temp dir: %w", err)
} }
defer func() { _ = os.RemoveAll(tmpDir) }() defer os.RemoveAll(tmpDir)
// Set restrictive permissions // Set restrictive permissions
if err := os.Chmod(tmpDir, 0o700); err != nil { if err := os.Chmod(tmpDir, 0o700); err != nil {
@@ -184,7 +184,7 @@ func gpgVerify(data, signature, pubKey []byte) error {
} }
// Import the public key into the temporary keyring // Import the public key into the temporary keyring
importCmd := exec.Command("gpg", "--batch", "--no-tty", importCmd := exec.Command("gpg",
"--homedir", tmpDir, "--homedir", tmpDir,
"--import", "--import",
pubKeyFile, pubKeyFile,
@@ -196,7 +196,7 @@ func gpgVerify(data, signature, pubKey []byte) error {
} }
// Verify the signature // Verify the signature
verifyCmd := exec.Command("gpg", "--batch", "--no-tty", verifyCmd := exec.Command("gpg",
"--homedir", tmpDir, "--homedir", tmpDir,
"--verify", "--verify",
sigFile, sigFile,

View File

@@ -34,15 +34,15 @@ func testGPGEnv(t *testing.T) (GPGKeyID, func()) {
// Save original GNUPGHOME and set new one // Save original GNUPGHOME and set new one
origGPGHome := os.Getenv("GNUPGHOME") origGPGHome := os.Getenv("GNUPGHOME")
require.NoError(t, os.Setenv("GNUPGHOME", gpgHome)) os.Setenv("GNUPGHOME", gpgHome)
cleanup := func() { cleanup := func() {
if origGPGHome == "" { if origGPGHome == "" {
_ = os.Unsetenv("GNUPGHOME") os.Unsetenv("GNUPGHOME")
} else { } else {
_ = os.Setenv("GNUPGHOME", origGPGHome) os.Setenv("GNUPGHOME", origGPGHome)
} }
_ = os.RemoveAll(gpgHome) os.RemoveAll(gpgHome)
} }
// Generate a test key with no passphrase // Generate a test key with no passphrase

View File

@@ -17,7 +17,6 @@ type manifest struct {
pbOuter *MFFileOuter pbOuter *MFFileOuter
output *bytes.Buffer output *bytes.Buffer
signingOptions *SigningOptions signingOptions *SigningOptions
fixedUUID []byte // if set, use this UUID instead of generating one
} }
func (m *manifest) String() string { func (m *manifest) String() string {

View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.36.11 // protoc-gen-go v1.36.11
// protoc v6.33.4 // protoc v6.33.0
// source: mf.proto // source: mf.proto
package mfer package mfer
@@ -329,9 +329,6 @@ func (x *MFFileOuter) GetSigningPubKey() []byte {
type MFFilePath struct { type MFFilePath struct {
state protoimpl.MessageState `protogen:"open.v1"` state protoimpl.MessageState `protogen:"open.v1"`
// required attributes: // required attributes:
// Path invariants: must be valid UTF-8, use forward slashes only,
// be relative (no leading /), contain no ".." segments, and no
// empty segments (no "//").
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
// gotta have at least one: // gotta have at least one:
@@ -340,6 +337,7 @@ type MFFilePath struct {
MimeType *string `protobuf:"bytes,301,opt,name=mimeType,proto3,oneof" json:"mimeType,omitempty"` MimeType *string `protobuf:"bytes,301,opt,name=mimeType,proto3,oneof" json:"mimeType,omitempty"`
Mtime *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"` Mtime *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"`
Ctime *Timestamp `protobuf:"bytes,303,opt,name=ctime,proto3,oneof" json:"ctime,omitempty"` Ctime *Timestamp `protobuf:"bytes,303,opt,name=ctime,proto3,oneof" json:"ctime,omitempty"`
Atime *Timestamp `protobuf:"bytes,304,opt,name=atime,proto3,oneof" json:"atime,omitempty"`
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
} }
@@ -416,6 +414,13 @@ func (x *MFFilePath) GetCtime() *Timestamp {
return nil return nil
} }
func (x *MFFilePath) GetAtime() *Timestamp {
if x != nil {
return x.Atime
}
return nil
}
type MFFileChecksum struct { type MFFileChecksum struct {
state protoimpl.MessageState `protogen:"open.v1"` state protoimpl.MessageState `protogen:"open.v1"`
// 1.0 golang implementation must write a multihash here // 1.0 golang implementation must write a multihash here
@@ -561,7 +566,7 @@ const file_mf_proto_rawDesc = "" +
"\n" + "\n" +
"_signatureB\t\n" + "_signatureB\t\n" +
"\a_signerB\x10\n" + "\a_signerB\x10\n" +
"\x0e_signingPubKey\"\xf0\x01\n" + "\x0e_signingPubKey\"\xa2\x02\n" +
"\n" + "\n" +
"MFFilePath\x12\x12\n" + "MFFilePath\x12\x12\n" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" +
@@ -571,10 +576,13 @@ const file_mf_proto_rawDesc = "" +
"\x05mtime\x18\xae\x02 \x01(\v2\n" + "\x05mtime\x18\xae\x02 \x01(\v2\n" +
".TimestampH\x01R\x05mtime\x88\x01\x01\x12&\n" + ".TimestampH\x01R\x05mtime\x88\x01\x01\x12&\n" +
"\x05ctime\x18\xaf\x02 \x01(\v2\n" + "\x05ctime\x18\xaf\x02 \x01(\v2\n" +
".TimestampH\x02R\x05ctime\x88\x01\x01B\v\n" + ".TimestampH\x02R\x05ctime\x88\x01\x01\x12&\n" +
"\x05atime\x18\xb0\x02 \x01(\v2\n" +
".TimestampH\x03R\x05atime\x88\x01\x01B\v\n" +
"\t_mimeTypeB\b\n" + "\t_mimeTypeB\b\n" +
"\x06_mtimeB\b\n" + "\x06_mtimeB\b\n" +
"\x06_ctime\".\n" + "\x06_ctimeB\b\n" +
"\x06_atime\".\n" +
"\x0eMFFileChecksum\x12\x1c\n" + "\x0eMFFileChecksum\x12\x1c\n" +
"\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xd6\x01\n" + "\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xd6\x01\n" +
"\x06MFFile\x12)\n" + "\x06MFFile\x12)\n" +
@@ -619,14 +627,15 @@ var file_mf_proto_depIdxs = []int32{
6, // 2: MFFilePath.hashes:type_name -> MFFileChecksum 6, // 2: MFFilePath.hashes:type_name -> MFFileChecksum
3, // 3: MFFilePath.mtime:type_name -> Timestamp 3, // 3: MFFilePath.mtime:type_name -> Timestamp
3, // 4: MFFilePath.ctime:type_name -> Timestamp 3, // 4: MFFilePath.ctime:type_name -> Timestamp
2, // 5: MFFile.version:type_name -> MFFile.Version 3, // 5: MFFilePath.atime:type_name -> Timestamp
5, // 6: MFFile.files:type_name -> MFFilePath 2, // 6: MFFile.version:type_name -> MFFile.Version
3, // 7: MFFile.createdAt:type_name -> Timestamp 5, // 7: MFFile.files:type_name -> MFFilePath
8, // [8:8] is the sub-list for method output_type 3, // 8: MFFile.createdAt:type_name -> Timestamp
8, // [8:8] is the sub-list for method input_type 9, // [9:9] is the sub-list for method output_type
8, // [8:8] is the sub-list for extension type_name 9, // [9:9] is the sub-list for method input_type
8, // [8:8] is the sub-list for extension extendee 9, // [9:9] is the sub-list for extension type_name
0, // [0:8] is the sub-list for field type_name 9, // [9:9] is the sub-list for extension extendee
0, // [0:9] is the sub-list for field type_name
} }
func init() { file_mf_proto_init() } func init() { file_mf_proto_init() }

View File

@@ -59,6 +59,7 @@ message MFFilePath {
optional string mimeType = 301; optional string mimeType = 301;
optional Timestamp mtime = 302; optional Timestamp mtime = 302;
optional Timestamp ctime = 303; optional Timestamp ctime = 303;
optional Timestamp atime = 304;
} }
message MFFileChecksum { message MFFileChecksum {

View File

@@ -43,12 +43,10 @@ type ScanStatus struct {
// ScannerOptions configures scanner behavior. // ScannerOptions configures scanner behavior.
type ScannerOptions struct { type ScannerOptions struct {
IncludeDotfiles bool // Include files and directories starting with a dot (default: exclude) IncludeDotfiles bool // Include files and directories starting with a dot (default: exclude)
FollowSymLinks bool // Resolve symlinks instead of skipping them FollowSymLinks bool // Resolve symlinks instead of skipping them
IncludeTimestamps bool // Include createdAt timestamp in manifest (default: omit for determinism) Fs afero.Fs // Filesystem to use, defaults to OsFs if nil
Fs afero.Fs // Filesystem to use, defaults to OsFs if nil SigningOptions *SigningOptions // GPG signing options (nil = no signing)
SigningOptions *SigningOptions // GPG signing options (nil = no signing)
Seed string // If set, derive a deterministic UUID from this seed
} }
// FileEntry represents a file that has been enumerated. // FileEntry represents a file that has been enumerated.
@@ -275,15 +273,9 @@ func (s *Scanner) ToManifest(ctx context.Context, w io.Writer, progress chan<- S
s.mu.RUnlock() s.mu.RUnlock()
builder := NewBuilder() builder := NewBuilder()
if s.options.IncludeTimestamps {
builder.SetIncludeTimestamps(true)
}
if s.options.SigningOptions != nil { if s.options.SigningOptions != nil {
builder.SetSigningOptions(s.options.SigningOptions) builder.SetSigningOptions(s.options.SigningOptions)
} }
if s.options.Seed != "" {
builder.SetSeed(s.options.Seed)
}
var scannedFiles FileCount var scannedFiles FileCount
var scannedBytes FileSize var scannedBytes FileSize
@@ -393,9 +385,6 @@ func (s *Scanner) ToManifest(ctx context.Context, w io.Writer, progress chan<- S
// The path should use forward slashes. // The path should use forward slashes.
func IsHiddenPath(p string) bool { func IsHiddenPath(p string) bool {
tp := path.Clean(p) tp := path.Clean(p)
if tp == "." || tp == "/" {
return false
}
if strings.HasPrefix(tp, ".") { if strings.HasPrefix(tp, ".") {
return true return true
} }

View File

@@ -352,10 +352,6 @@ func TestIsHiddenPath(t *testing.T) {
{"/absolute/.hidden", true}, {"/absolute/.hidden", true},
{"./relative", false}, // path.Clean removes leading ./ {"./relative", false}, // path.Clean removes leading ./
{"a/b/c/.d/e", true}, {"a/b/c/.d/e", true},
{".", false}, // current directory is not hidden (#14)
{"/", false}, // root is not hidden
{"./", false}, // current directory with trailing slash
{"./file.txt", false}, // file in current directory
} }
for _, tt := range tests { for _, tt := range tests {

View File

@@ -34,12 +34,12 @@ func (m *manifest) generate() error {
} }
dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter) dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
if err != nil { if err != nil {
return fmt.Errorf("serialize: marshal outer: %w", err) return err
} }
m.output = bytes.NewBuffer([]byte(MAGIC)) m.output = bytes.NewBuffer([]byte(MAGIC))
_, err = m.output.Write(dat) _, err = m.output.Write(dat)
if err != nil { if err != nil {
return fmt.Errorf("serialize: write output: %w", err) return err
} }
return nil return nil
} }
@@ -49,29 +49,24 @@ func (m *manifest) generateOuter() error {
return errors.New("internal error") return errors.New("internal error")
} }
// Use fixed UUID if provided, otherwise generate a new one // Generate UUID and set on inner message
var manifestUUID uuid.UUID manifestUUID := uuid.New()
if len(m.fixedUUID) == 16 {
copy(manifestUUID[:], m.fixedUUID)
} else {
manifestUUID = uuid.New()
}
m.pbInner.Uuid = manifestUUID[:] m.pbInner.Uuid = manifestUUID[:]
innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner) innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
if err != nil { if err != nil {
return fmt.Errorf("serialize: marshal inner: %w", err) return err
} }
// Compress the inner data // Compress the inner data
idc := new(bytes.Buffer) idc := new(bytes.Buffer)
zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
if err != nil { if err != nil {
return fmt.Errorf("serialize: create compressor: %w", err) return err
} }
_, err = zw.Write(innerData) _, err = zw.Write(innerData)
if err != nil { if err != nil {
return fmt.Errorf("serialize: compress: %w", err) return err
} }
_ = zw.Close() _ = zw.Close()
@@ -80,7 +75,7 @@ func (m *manifest) generateOuter() error {
// Hash the compressed data for integrity verification before decompression // Hash the compressed data for integrity verification before decompression
h := sha256.New() h := sha256.New()
if _, err := h.Write(compressedData); err != nil { if _, err := h.Write(compressedData); err != nil {
return fmt.Errorf("serialize: hash write: %w", err) return err
} }
sha256Hash := h.Sum(nil) sha256Hash := h.Sum(nil)

View File

@@ -27,12 +27,8 @@ func (b BaseURL) JoinPath(path RelFilePath) (FileURL, error) {
base.Path += "/" base.Path += "/"
} }
// Encode each path segment individually to preserve slashes // Parse and encode the relative path
segments := strings.Split(string(path), "/") ref, err := url.Parse(url.PathEscape(string(path)))
for i, seg := range segments {
segments[i] = url.PathEscape(seg)
}
ref, err := url.Parse(strings.Join(segments, "/"))
if err != nil { if err != nil {
return "", err return "", err
} }

View File

@@ -1,44 +0,0 @@
package mfer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBaseURLJoinPath(t *testing.T) {
tests := []struct {
base BaseURL
path RelFilePath
expected string
}{
{"https://example.com/dir/", "file.txt", "https://example.com/dir/file.txt"},
{"https://example.com/dir", "file.txt", "https://example.com/dir/file.txt"},
{"https://example.com/", "sub/file.txt", "https://example.com/sub/file.txt"},
{"https://example.com/dir/", "file with spaces.txt", "https://example.com/dir/file%20with%20spaces.txt"},
}
for _, tt := range tests {
t.Run(string(tt.base)+"+"+string(tt.path), func(t *testing.T) {
result, err := tt.base.JoinPath(tt.path)
require.NoError(t, err)
assert.Equal(t, tt.expected, string(result))
})
}
}
func TestBaseURLString(t *testing.T) {
b := BaseURL("https://example.com/")
assert.Equal(t, "https://example.com/", b.String())
}
func TestFileURLString(t *testing.T) {
f := FileURL("https://example.com/file.txt")
assert.Equal(t, "https://example.com/file.txt", f.String())
}
func TestManifestURLString(t *testing.T) {
m := ManifestURL("https://example.com/index.mf")
assert.Equal(t, "https://example.com/index.mf", m.String())
}

BIN
modcache.tzst Normal file

Binary file not shown.

View File

@@ -1,155 +0,0 @@
#!/bin/sh
# script/bootstrap: install all dependencies needed to build and develop
# this repo. Idempotent: every install is guarded by a check so already
# installed tools are skipped. Base tooling comes from nix, apt, brew,
# or apk (detected in that order); assumes NOTHING is present (not git,
# make, node, yarn, go, or python). Node is used directly if installed;
# otherwise a pinned version is installed via nvm (installing nvm
# itself first, from a hash-verified release archive, never curl | sh).
#
# Uncomment the language sections in main() that apply to this repo.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions.
NODE_VERSION="22.17.0"
NVM_VERSION="0.40.3"
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
YARN_VERSION="1.22.22"
PKGMGR=""
SUDO=""
detect_pkgmgr() {
[ -n "$PKGMGR" ] && return 0
if command -v nix-env >/dev/null 2>&1; then
PKGMGR="nix"
elif command -v apt-get >/dev/null 2>&1; then
PKGMGR="apt"
elif command -v brew >/dev/null 2>&1; then
PKGMGR="brew"
elif command -v apk >/dev/null 2>&1; then
PKGMGR="apk"
else
echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2
exit 1
fi
if [ "$PKGMGR" = "apt" ]; then
export DEBIAN_FRONTEND=noninteractive
if [ "$(id -u)" != "0" ]; then
SUDO="sudo"
fi
fi
}
# pkg_install <nix-attr> <apt-pkg> <brew-formula> <apk-pkg>
pkg_install() {
detect_pkgmgr
case "$PKGMGR" in
nix) nix-env -iA "nixpkgs.$1" ;;
apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" ;;
brew) brew install "$3" ;;
apk) apk add --no-cache "$4" ;;
esac
}
missing() {
! command -v "$1" >/dev/null 2>&1
}
# verify_sha256 <file> <expected-hash>
verify_sha256() {
if command -v sha256sum >/dev/null 2>&1; then
actual="$(sha256sum "$1" | cut -d' ' -f1)"
else
actual="$(shasum -a 256 "$1" | cut -d' ' -f1)"
fi
if [ "$actual" != "$2" ]; then
echo "bootstrap: sha256 mismatch for $1" >&2
echo " expected: $2" >&2
echo " actual: $actual" >&2
exit 1
fi
}
# nvm is a bash script; run a command in a bash with nvm loaded
nvm_sh() {
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
}
ensure_nvm() {
[ -s "$HOME/.nvm/nvm.sh" ] && return 0
# nvm prerequisites; nvm itself requires bash, so install it too
if missing bash; then pkg_install bash bash bash bash; fi
if missing curl; then pkg_install curl curl curl curl; fi
if missing git; then pkg_install git git git git; fi
tmp="$(mktemp -d)"
curl -fsSL -o "$tmp/nvm.tar.gz" \
"https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz"
verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256"
mkdir -p "$HOME/.nvm"
tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1
rm -rf "$tmp"
}
ensure_node() {
if ! missing node; then return 0; fi
ensure_nvm
nvm_sh "nvm install $NODE_VERSION"
}
ensure_yarn() {
if ! missing yarn; then return 0; fi
if ! missing corepack; then
corepack enable
corepack prepare "yarn@$YARN_VERSION" --activate
elif [ -s "$HOME/.nvm/nvm.sh" ]; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
corepack prepare yarn@$YARN_VERSION --activate"
else
npm install -g "yarn@$YARN_VERSION"
fi
}
install_js_deps() {
if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
yarn install --frozen-lockfile"
else
yarn install --frozen-lockfile
fi
}
main() {
cd "$ROOT"
# Base tooling (every repo)
if missing git; then pkg_install git git git git; fi
if missing make; then pkg_install gnumake make make make; fi
# ---- JS / docs repos ----
# ensure_node
# ensure_yarn
# install_js_deps
# ---- Go repos ----
if missing go; then pkg_install go golang go go; fi
# golangci-lint: packaged in nix, brew, and apk. On apt there is no
# package: download a specific release archive from GitHub and
# verify its hash (verify_sha256), never curl | sh.
if missing golangci-lint; then
pkg_install golangci-lint golangci-lint golangci-lint golangci-lint
fi
go mod download
# ---- Python repos ----
# if missing python3; then pkg_install python3 python3 python3 python3; fi
# python3 -m venv .venv
# ./.venv/bin/pip install -e '.[dev]'
echo "bootstrap complete"
}
main "$@"

View File

@@ -1,15 +0,0 @@
#!/bin/sh
# script/check: run all checks (test, lint, fmt-check). Our own
# extension to scripts-to-rule-them-all. Must not modify any files.
# Generic: usually needs no adaptation.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
main() {
"$SCRIPT_DIR/test"
"$SCRIPT_DIR/lint"
"$SCRIPT_DIR/fmt-check"
}
main "$@"

View File

@@ -1,14 +0,0 @@
#!/bin/sh
# script/cibuild: run the CI build. The Dockerfile runs script/check
# (via make check), so a successful build implies all checks pass.
# Generic: needs no adaptation. The Gitea workflow runs this on push.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
docker build .
}
main "$@"

View File

@@ -1,15 +0,0 @@
#!/bin/sh
# script/docker: build the Docker image tagged with the project name.
# Identical in all repos; the tag comes from script/projectname.
# Generic: needs no adaptation.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
docker build -t "$("$SCRIPT_DIR/projectname")" .
}
main "$@"

View File

@@ -1,27 +0,0 @@
#!/bin/sh
# script/fmt: format all files (writes).
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
# (mirrors the old Makefile prerequisite; the generated file is
# committed, so this is normally a no-op).
ensure_pb() {
if [ ! -f mfer/mf.pb.go ] ||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
(cd mfer && go generate .)
fi
}
main() {
cd "$ROOT"
ensure_pb
gofumpt -l -w mfer internal cmd
golangci-lint run --fix
# prettier is best-effort, as in the old Makefile (- prefix)
prettier -w *.json || true
prettier -w *.md || true
}
main "$@"

View File

@@ -1,28 +0,0 @@
#!/bin/sh
# script/fmt-check: check formatting (read-only). Same scope as
# script/fmt, but fails instead of writing.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
# (mirrors the old Makefile prerequisite; the generated file is
# committed, so this is normally a no-op).
ensure_pb() {
if [ ! -f mfer/mf.pb.go ] ||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
(cd mfer && go generate .)
fi
}
main() {
cd "$ROOT"
ensure_pb
if [ -n "$(gofmt -l .)" ]; then
echo "gofmt: files need formatting:" >&2
gofmt -l . >&2
exit 1
fi
}
main "$@"

View File

@@ -1,17 +0,0 @@
#!/bin/sh
# script/install-precommit: install the git pre-commit hook that runs
# script/precommit. Our own extension to scripts-to-rule-them-all.
# Generic: needs no adaptation.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
hook=".git/hooks/pre-commit"
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit
chmod +x .git/hooks/pre-commit
echo "pre-commit hook installed: runs script/precommit"
}
main "$@"

View File

@@ -1,17 +0,0 @@
#!/bin/sh
# script/lint: run the linter.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
golangci-lint run
if [ -n "$(gofmt -l .)" ]; then
echo "gofmt: files need formatting:" >&2
gofmt -l . >&2
exit 1
fi
}
main "$@"

View File

@@ -1,20 +0,0 @@
#!/bin/sh
# script/precommit: run by the git pre-commit hook; fails the commit if
# checks fail. Our own extension to scripts-to-rule-them-all. Go repo
# extras run first: go mod tidy and go fmt, failing the commit if they
# change go.mod or go.sum.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
go mod tidy
go fmt ./...
git diff --exit-code -- go.mod go.sum ||
{ echo "go mod tidy changed files; stage and retry" >&2; exit 1; }
"$SCRIPT_DIR/check"
}
main "$@"

View File

@@ -1,12 +0,0 @@
#!/bin/sh
# script/projectname: output the name of this project. Our own
# extension to scripts-to-rule-them-all. Other scripts that need the
# name (e.g. script/docker) call this, so they can stay identical
# across all repos.
set -eu
main() {
echo "mfer"
}
main "$@"

View File

@@ -1,14 +0,0 @@
#!/bin/sh
# script/setup: set up the repo for development after a fresh clone:
# installs dependencies (script/bootstrap) and the git pre-commit hook.
# Add any repo-specific initialization (db init, .env template) here.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
main() {
"$SCRIPT_DIR/bootstrap"
"$SCRIPT_DIR/install-precommit"
}
main "$@"

View File

@@ -1,23 +0,0 @@
#!/bin/sh
# script/test: run the test suite.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
# (mirrors the old Makefile prerequisite; the generated file is
# committed, so this is normally a no-op).
ensure_pb() {
if [ ! -f mfer/mf.pb.go ] ||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
(cd mfer && go generate .)
fi
}
main() {
cd "$ROOT"
ensure_pb
go test -v --timeout 10s ./...
}
main "$@"

BIN
vendor.tzst Normal file

Binary file not shown.