feat: split Dockerfile into dedicated lint stage

Add pre-built golangci/golangci-lint:v1.64.8 image as a separate lint
stage for fast lint feedback without rebuilding the entire builder.

- Lint stage: installs protobuf-compiler, generates code, runs make lint
- Build stage: pins sneak/builder by sha256, removes inline golangci-lint
- COPY --from=lint forces BuildKit to execute the lint stage
- All images pinned by sha256 digest
- Final stage (FROM scratch) unchanged

.dockerignore

@@ -1,7 +1,3 @@
-*.tzst
-*.tar
-/buildimage
-/dockerdeps
-/tmp
-*.docker.tzst
 *.tmp
+*.dockerimage
+.git

.drone.yml

@@ -7,8 +7,17 @@ steps:
   network_mode: bridge
   settings:
     repo: sneak/mfer
+    build_args_from_env: [ DRONE_COMMIT_SHA ]
     dry_run: true
     custom_dns: [ 116.202.204.30 ]
     tags:
-      - ${DRONE_COMMIT_SHA}
+      - ${DRONE_COMMIT_SHA:0:7}
       - ${DRONE_BRANCH}
+      - latest
+- name: notify
+  image: plugins/slack
+  settings:
+    webhook:
+      from_secret: SLACK_WEBHOOK_URL
+  when:
+    event: pull_request

.gitignore

@@ -1,8 +1,6 @@
 mfer/*.pb.go
 /mfer.cmd
-vendor
 /tmp
 *.tmp
-*.docker.tzst
+*.dockerimage
-*.tzst
+/vendor
-/builddeps/modcache.tar

Dockerfile

@@ -1,46 +1,47 @@
 ################################################################################
 #2345678911234567892123456789312345678941234567895123456789612345678971234567898
 ################################################################################
-FROM golang:1.19.3-bullseye AS builder
+# Lint stage — fast feedback on formatting and lint issues
-ENV GOPATH /go
+# golangci/golangci-lint:v1.64.8
+FROM golangci/golangci-lint@sha256:2987913e27f4eca9c8a39129d2c7bc1e74fbcf77f181e01cea607be437aa5cb8 AS lint
+WORKDIR /src
+RUN apt-get update && apt-get install -y --no-install-recommends protobuf-compiler && rm -rf /var/lib/apt/lists/*
+COPY go.mod go.sum ./
+RUN go mod download
+RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
+COPY . .
+RUN cd mfer && go generate .
+RUN make lint
+################################################################################
+#2345678911234567892123456789312345678941234567895123456789612345678971234567898
+################################################################################
+# Build stage
+# sneak/builder:2022-12-08
+FROM sneak/builder@sha256:d61175320b02b04eb2cada3005833494ce8a4032647e4202dcb3551cf0b56f5a AS builder
 ENV DEBIAN_FRONTEND noninteractive
-RUN --mount=type=cache,target=/var/cache/apt \
-    apt update && \
-    apt install -y make zstd unzip && \
-    mkdir /build
-WORKDIR /tmp
-# install newer protoc
-COPY ./builddeps/* ./
-RUN unzip protoc-*-linux-$(uname -m).zip -d /usr/local && \
-    protoc --version && \
-    mkdir -p /go/pkg/mod && \
-    cd /go/pkg/mod && \
-    tar xvf /tmp/modcache.tar && \
-    cd / && \
-    go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 && \
-    go install -v mvdan.cc/gofumpt@latest && \
-    go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 && \
-    rm -rf /tmp/*
 WORKDIR /build
-COPY ./go.mod ./go.sum .
+# Force BuildKit to run the lint stage by creating a stage dependency
-RUN \
+COPY --from=lint /src/go.sum /dev/null
-    go mod download -x
+COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum /build/
-################################################################################
+COPY ./vendor.tzst /build/vendor.tzst
-#### caching phase done
+COPY ./modcache.tzst /build/modcache.tzst
-################################################################################
-WORKDIR /build
-COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum .
 COPY ./internal ./internal
 COPY ./bin/gitrev.sh ./bin/gitrev.sh
 COPY ./mfer ./mfer
 COPY ./cmd ./cmd
-RUN find /build
 ARG GITREV unknown
+ARG DRONE_COMMIT_SHA unknown
+
+RUN mkdir -p "$(go env GOMODCACHE)" && cd "$(go env GOMODCACHE)" && \
+    zstdmt -d --stdout /build/modcache.tzst | tar xf - && \
+    rm /build/modcache.tzst && cd /build
 RUN \
     cd mfer && go generate . && cd .. && \
-    GOPACKAGESDEBUG=true golangci-lint run ./... && \
+    mkdir vendor && cd vendor && \
+    zstdmt -d --stdout /build/vendor.tzst | tar xf - && rm /build/vendor.tzst && \
+    cd .. && \
     make mfer.cmd
-RUN go mod vendor && tar -c . | zstdmt -19 > /src.tzst
+RUN rm -rf /build/vendor && go mod vendor && tar -c . | zstdmt -19 > /src.tzst
 ################################################################################
 #2345678911234567892123456789312345678941234567895123456789612345678971234567898
 ################################################################################

Makefile

@@ -44,9 +44,9 @@ mfer.cmd: $(SOURCEFILES) mfer/mf.pb.go
 	cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../mfer.cmd $(GOFLAGS) .
 
 clean:
-	rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer
+	rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer *.dockerimage
 
-fmt:
+fmt: mfer/mf.pb.go
 	gofumpt -l -w mfer internal cmd
 	golangci-lint run --fix
 	-prettier -w *.json
@@ -56,10 +56,9 @@ lint:
 	golangci-lint run
 	sh -c 'test -z "$$(gofmt -l .)"'
 
-docker: sneak-mfer.$(ARCH).docker.tzst
+docker: sneak-mfer.$(ARCH).tzst.dockerimage
 
-sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES)
+sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
-	bash -x bin/docker-prereqs.sh
 	docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
 	docker save sneak/mfer | pv | zstdmt -19 > $@
 	du -sh $@
@@ -67,3 +66,17 @@ sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES)
 godoc:
 	open http://127.0.0.1:6060
 	godoc -http=:6060
+
+vendor.tzst: go.mod go.sum
+	go mod tidy
+	go mod vendor
+	cd vendor && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
+	rm -rf vendor
+	mv $@.tmp $@
+
+modcache.tzst: go.mod go.sum
+	go mod tidy
+	cd $(HOME)/go/pkg && chmod -R u+rw . && rm -rf mod sumdb
+	go mod download -x
+	cd $(shell go env GOMODCACHE) && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
+	mv $@.tmp $@

README.md

@@ -1,11 +1,48 @@
 # mfer
 
-Manifest file generator and checker.
+[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library
+and thin wrapper command-line utility written in [Go](https://golang.org)
+and first published in 2022 under the [WTFPL](https://wtfpl.net) (public
+domain) license.  It specifies and generates `.mf` manifest files over a
+directory tree of files to encapsulate metadata about them (such as
+cryptographic checksums or signatures over same) to aid in archiving,
+downloading, and streaming, or mirroring.  The manifest files' data is
+serialized with Google's [protobuf serialization
+format](https://developers.google.com/protocol-buffers).  The structure of
+these files can be found [in the format
+specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
+which is included in the [project
+repository](https://git.eeqj.de/sneak/mfer).
+
+The current version is pre-1.0 and while the repo was published in 2022,
+there has not yet been any versioned release.  [SemVer](https://semver.org)
+will be used for releases.
+
+This project was started by [@sneak](https://sneak.berlin) to scratch an
+itch in 2022 and is currently a one-person effort, though the goal is for
+this to emerge as a de-facto standard and be incorporated into other
+software.  A compatible javascript library is planned.
 
 # Build Status
 
 [![Build Status](https://drone.datavi.be/api/badges/sneak/mfer/status.svg)](https://drone.datavi.be/sneak/mfer)
 
+# Participation
+
+The community is as yet nonexistent so there are no defined policies or
+norms yet.  Primary development happens on a privately-run Gitea instance at
+[https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) and issues
+are [tracked there](https://git.eeqj.de/sneak/mfer/issues).
+
+Changes must always be formatted with a standard `go fmt`, syntactically
+valid, and must pass the linting defined in the repository (presently only
+the `golangci-lint` defaults), which can be run with a `make lint`.  The
+`main` branch is protected and all changes must be made via [pull
+requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
+Any changes submitted to this project must also be
+[WTFPL-licensed](https://wtfpl.net) to be considered.
+
+
 # Problem Statement
 
 Given a plain URL, there is no standard way to safely and programmatically
@@ -170,6 +207,24 @@ regardless of filesystem format.
 Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your
 desired username for an account on this Gitea instance.
 
-I am currently interested in hiring a contractor skilled with the Go
+# See Also
-standard library interfaces to specify this tool in full and develop a
+
-prototype implementation.
+## Prior Art: Metalink
+
+* [Metalink - Mozilla Wiki](https://wiki.mozilla.org/Metalink)
+* [Metalink - Wikipedia](https://en.wikipedia.org/wiki/Metalink)
+* [RFC 5854 - The Metalink Download Description Format](https://datatracker.ietf.org/doc/html/rfc5854)
+* [RFC 6249 - Metalink/HTTP: Mirrors and Hashes](https://www.rfc-editor.org/rfc/rfc6249.html)
+
+## Links
+
+* Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
+* Issues: [https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
+
+# Authors
+
+* [@sneak <sneak@sneak.berlin>](mailto:sneak@sneak.berlin)
+
+# License
+
+* [WTFPL](https://wtfpl.net)

TODO.md

@@ -0,0 +1,122 @@
+# TODO: mfer 1.0
+
+## Design Questions
+
+*sneak: please answer inline below each question. These are preserved for posterity.*
+
+### Format Design
+
+**1. Should `MFFileChecksum` be simplified?**
+Currently it's a separate message wrapping a single `bytes multiHash` field. Since multihash already self-describes the algorithm, `repeated bytes hashes` directly on `MFFilePath` would be simpler and reduce per-file protobuf overhead. Is the extra message layer intentional (e.g. planning to add per-hash metadata like `verified_at`)?
+
+> *answer:*
+
+**2. Should file permissions/mode be stored?**
+The format stores mtime/ctime but not Unix file permissions. For archival use (ExFAT, filesystem-independent checksums) this may not matter, but for software distribution or filesystem restoration it's a gap. Should we reserve a field now (e.g. `optional uint32 mode = 305`) even if we don't populate it yet?
+
+> *answer:*
+
+**3. Should `atime` be removed from the schema?**
+Access time is volatile, non-deterministic, and often disabled (`noatime`). Including it means two manifests of the same directory at different times will differ, which conflicts with the determinism goal. Remove it, or document it as "never set by default"?
+
+> *answer:*
+
+**4. What are the path normalization rules?**
+The proto has `string path` with no specification about: always forward-slash? Must be relative? No `..` components allowed? UTF-8 NFC vs NFD normalization (macOS vs Linux)? Max path length? This is a security issue (path traversal) and a cross-platform compatibility issue. What rules should the spec mandate?
+
+> *answer:*
+
+**5. Should we add a version byte after the magic?**
+Currently `ZNAVSRFG` is followed immediately by protobuf. Adding a version byte (`ZNAVSRFG\x01`) would allow future framing changes without requiring protobuf parsing to detect the version. `MFFileOuter.Version` serves this purpose but requires successful deserialization to read. Worth the extra byte?
+
+> *answer:*
+
+**6. Should we add a length-prefix after the magic?**
+Protobuf is not self-delimiting. If we ever want to concatenate manifests or append data after the protobuf, the current framing is insufficient. Add a varint or fixed-width length-prefix?
+
+> *answer:*
+
+### Signature Design
+
+**7. What does the outer SHA-256 hash cover — compressed or uncompressed data?**
+The review notes it currently hashes compressed data (good for verifying before decompression), but this should be explicitly documented. Which is the intended behavior?
+
+> *answer:*
+
+**8. Should `signatureString()` sign raw bytes instead of a hex-encoded string?**
+Currently the canonical string is `MAGIC-UUID-MULTIHASH` with hex encoding, which adds a transformation layer. Signing the raw `sha256` bytes (or compressed `innerMessage` directly) would be simpler. Keep the string format or switch to raw bytes?
+
+> *answer:*
+
+**9. Should we support detached signature files (`.mf.sig`)?**
+Embedded signatures are better for single-file distribution. Detached `.mf.sig` files follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are simpler for HTTP serving. Support both modes?
+
+> *answer:*
+
+**10. GPG vs pure-Go crypto for signatures?**
+Shelling out to `gpg` is fragile (may not be installed, version-dependent output). `github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could go Ed25519/signify (simpler, no key management). Which direction?
+
+> *answer:*
+
+### Implementation Design
+
+**11. Should manifests be deterministic by default?**
+This means: sort file entries by path, omit `createdAt` timestamp (or make it opt-in), no `atime`. Should determinism be the default, with a `--include-timestamps` flag to opt in?
+
+> *answer:*
+
+**12. Should we consolidate or keep both scanner/checker implementations?**
+There are two parallel implementations: `mfer/scanner.go` + `mfer/checker.go` (typed with `FileSize`, `RelFilePath`) and `internal/scanner/` + `internal/checker/` (raw `int64`, `string`). The `mfer/` versions are superior. Delete the `internal/` versions?
+
+> *answer:*
+
+**13. Should the `manifest` type be exported?**
+Currently unexported with exported constructors (`New`, `NewFromPaths`, etc.). Consumers can't declare `var m *mfer.manifest`. Export the type, or define an interface?
+
+> *answer:*
+
+**14. What should the Go module path be for 1.0?**
+Currently mixed between `sneak.berlin/go/mfer` and `git.eeqj.de/sneak/mfer`. Which is canonical?
+
+> *answer:*
+
+---
+
+## Implementation Plan
+
+### Phase 1: Foundation (format correctness)
+
+- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate on `mfer/` package versions; update CLI code
+- [ ] Add deterministic file ordering — sort entries by path (lexicographic, byte-order) in `Builder.Build()`; add test asserting byte-identical output from two runs
+- [ ] Add decompression size limit — `io.LimitReader` in `deserializeInner()` with `m.pbOuter.Size` as bound
+- [ ] Fix `errors.Is` dead code in checker — replace with `os.IsNotExist(err)` or `errors.Is(err, fs.ErrNotExist)`
+- [ ] Fix `AddFile` to verify size — check `totalRead == size` after reading, return error on mismatch
+- [ ] Specify path invariants — add proto comments (UTF-8, forward-slash, relative, no `..`, no leading `/`); validate in `Builder.AddFile` and `Builder.AddFileWithHash`
+
+### Phase 2: CLI polish
+
+- [ ] Fix flag naming — all CLI flags use kebab-case as primary (`--include-dotfiles`, `--follow-symlinks`)
+- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or `url.JoinPath()` instead of string concatenation
+- [ ] Add progress rate-limiting to Checker — throttle to once per second, matching Scanner
+- [ ] Add `--deterministic` flag (or make it default) — omit `createdAt`, sort files
+
+### Phase 3: Robustness
+
+- [ ] Replace GPG subprocess with pure-Go crypto — `github.com/ProtonMail/go-crypto` or Ed25519/signify
+- [ ] Add timeout to any remaining subprocess calls
+- [ ] Add fuzzing tests for `NewManifestFromReader`
+- [ ] Add retry logic to fetch — exponential backoff for transient HTTP errors
+
+### Phase 4: Format finalization
+
+- [ ] Remove or deprecate `atime` from proto (pending design question answer)
+- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future file permissions
+- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version 1
+- [ ] Write format specification document — separate from README: magic, outer structure, compression, inner structure, path invariants, signature scheme, canonical serialization
+
+### Phase 5: Release prep
+
+- [ ] Finalize Go module path
+- [ ] Audit all error messages for consistency and helpfulness
+- [ ] Add `--version` output matching SemVer
+- [ ] Tag v1.0.0

bin/docker-prereqs.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-GOI="golang:1.19.3-bullseye"
-#CII="golangci/golangci-lint:v1.50.1"
-
-if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then
-    mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
-fi
-
-DICD="$DOCKER_IMAGE_CACHE_DIR"
-
-function buildImageCache() {
-    if [[ ! -e "$DICD/go.tzst" ]]; then
-        docker pull $GOI
-        docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \
-            mv $DICD/go.tzst.tmp $DICD/go.tzst
-    fi
-
-    #if [[ ! -e "$DICD/ci.tzst" ]]; then
-    #    docker pull $CII
-    #    docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \
-    #        mv $DICD/ci.tzst.tmp $DICD/ci.tzst
-    #fi
-}
-
-function loadImageCache() {
-   # zstdmt -d --stdout $DICD/ci.tzst | pv | docker load
-    zstdmt -d --stdout $DICD/go.tzst | pv | docker load
-
-    #docker image ls $CII
-    docker image ls $GOI
-
-}
-
-function writeModuleCache() {
-    if [[ ! -e $DICD/modcache.tar ]]; then
-        cd $(go env GOMODCACHE)
-        tar -c . | pv > $DICD/modcache.tar.tmp && mv $DICD/modcache.tar.tmp $DICD/modcache.tar
-        cd -
-    fi
-    cp -av $DICD/modcache.tar ./builddeps/modcache.tar.tmp && mv ./builddeps/modcache.tar.tmp ./builddeps/modcache.tar
-}
-
-buildImageCache
-loadImageCache
-writeModuleCache
-

bin/gitrev.sh

@@ -1,4 +1,10 @@
 #!/bin/bash
+#
+if [[ ! -z "$DRONE_COMMIT_SHA" ]]; then
+    echo "${DRONE_COMMIT_SHA:0:7}"
+    exit 0
+fi
+
 if [[ ! -z "$GITREV" ]]; then
     echo $GITREV
 else

buildimage/Dockerfile

@@ -1,21 +0,0 @@
-## build image:
-FROM golang:1.19.3-bullseye AS builder
-
-ENV DEBIAN_FRONTEND noninteractive
-RUN apt update && apt install -y make bzip2 curl unzip
-RUN mkdir -p /build
-WORKDIR /build
-
-# install newer protoc
-RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \
-    unzip *.zip -d /usr/local && rm -v *.zip && protoc --version
-
-RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
-
-RUN go env
-
-COPY ./go.mod .
-COPY ./go.sum .
-
-RUN --mount=type=cache,target=/go/pkg go mod download -x
-RUN rm -rfv /var/cache/* /var/tmp/*

go.mod

@@ -1,6 +1,6 @@
 module git.eeqj.de/sneak/mfer
 
-go 1.17
+go 1.22
 
 require (
 	github.com/apex/log v1.9.0
@@ -10,7 +10,6 @@ require (
 	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.23.6
 	google.golang.org/protobuf v1.28.1
-
 )
 
 require (
@@ -25,7 +24,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect

go.sum

@@ -61,8 +61,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -190,13 +188,9 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
@@ -221,8 +215,6 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
 github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
 github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
-github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
-github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
 github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
@@ -537,7 +529,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/bork/error.go

@@ -0,0 +1,15 @@
+package bork
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	ErrMissingMagic  = errors.New("missing magic bytes in file")
+	ErrFileTruncated = errors.New("file/stream is truncated abnormally")
+)
+
+func Newf(format string, args ...interface{}) error {
+	return fmt.Errorf(format, args...)
+}

internal/bork/error_test.go

@@ -0,0 +1,11 @@
+package bork
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuild(t *testing.T) {
+	assert.NotNil(t, ErrMissingMagic)
+}

internal/cli/misc.go

@@ -1,32 +0,0 @@
-package cli
-
-import "fmt"
-
-// FIXME make this write to a bytes.Buffer with fprintf
-func dumpByteSlice(b []byte) {
-	var a [16]byte
-	n := (len(b) + 15) &^ 15
-	for i := 0; i < n; i++ {
-		if i%16 == 0 {
-			fmt.Printf("%4d", i)
-		}
-		if i%8 == 0 {
-			fmt.Print(" ")
-		}
-		if i < len(b) {
-			fmt.Printf(" %02X", b[i])
-		} else {
-			fmt.Print("   ")
-		}
-		if i >= len(b) {
-			a[i%16] = ' '
-		} else if b[i] < 32 || b[i] > 126 {
-			a[i%16] = '.'
-		} else {
-			a[i%16] = b[i]
-		}
-		if i%16 == 15 {
-			fmt.Printf("  %s\n", string(a[:]))
-		}
-	}
-}

internal/log/log.go

@@ -1,6 +1,9 @@
 package log
 
 import (
+	"fmt"
+	"runtime"
+
 	"github.com/apex/log"
 	acli "github.com/apex/log/handlers/cli"
 	"github.com/davecgh/go-spew/spew"
@@ -25,13 +28,25 @@ func Init() {
 	log.SetLevel(log.InfoLevel)
 }
 
+func Debugf(format string, args ...interface{}) {
+	DebugReal(fmt.Sprintf(format, args...), 2)
+}
+
 func Debug(arg string) {
-	log.Debug(arg)
+	DebugReal(arg, 2)
+}
+
+func DebugReal(arg string, cs int) {
+	_, callerFile, callerLine, ok := runtime.Caller(cs)
+	if !ok {
+		return
+	}
+	tag := fmt.Sprintf("%s:%d: ", callerFile, callerLine)
+	log.Debug(tag + arg)
 }
 
 func Dump(args ...interface{}) {
-	str := spew.Sdump(args...)
+	DebugReal(spew.Sdump(args...), 2)
-	Debug(str)
 }
 
 func EnableDebugLogging() {

internal/log/log_test.go

@@ -0,0 +1,12 @@
+package log
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuild(t *testing.T) {
+	Init()
+	assert.True(t, true)
+}

mfer/deserialize.go

@@ -0,0 +1,89 @@
+package mfer
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"io"
+
+	"git.eeqj.de/sneak/mfer/internal/bork"
+	"git.eeqj.de/sneak/mfer/internal/log"
+	"google.golang.org/protobuf/proto"
+)
+
+func (m *manifest) validateProtoOuter() error {
+	if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
+		return errors.New("unknown version")
+	}
+	if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
+		return errors.New("unknown compression type")
+	}
+
+	bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
+
+	gzr, err := gzip.NewReader(bb)
+	if err != nil {
+		return err
+	}
+
+	dat, err := io.ReadAll(gzr)
+	defer gzr.Close()
+
+	if err != nil {
+		return err
+	}
+
+	isize := len(dat)
+	if int64(isize) != m.pbOuter.Size {
+		log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
+		return bork.ErrFileTruncated
+	}
+	log.Debugf("inner data size is %d", isize)
+	log.Dump(dat)
+	log.Dump(m.pbOuter.Sha256)
+	return nil
+}
+
+func validateMagic(dat []byte) bool {
+	ml := len([]byte(MAGIC))
+	if len(dat) < ml {
+		return false
+	}
+	got := dat[0:ml]
+	expected := []byte(MAGIC)
+	return bytes.Equal(got, expected)
+}
+
+func NewFromProto(input io.Reader) (*manifest, error) {
+	m := New()
+	dat, err := io.ReadAll(input)
+	if err != nil {
+		return nil, err
+	}
+	if !validateMagic(dat) {
+		return nil, errors.New("invalid file format")
+	}
+
+	// remove magic bytes prefix:
+	ml := len([]byte(MAGIC))
+	bb := bytes.NewBuffer(dat[ml:])
+	dat = bb.Bytes()
+
+	log.Dump(dat)
+
+	// deserialize:
+	m.pbOuter = new(MFFileOuter)
+	err = proto.Unmarshal(dat, m.pbOuter)
+
+	if err != nil {
+		return nil, err
+	}
+
+	ve := m.validateProtoOuter()
+	if ve != nil {
+		return nil, ve
+	}
+
+	// FIXME TODO deserialize inner
+	return m, nil
+}

mfer/example_test.go

@@ -0,0 +1,42 @@
+package mfer
+
+import (
+	"bytes"
+	"testing"
+
+	"git.eeqj.de/sneak/mfer/internal/log"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAPIExample(t *testing.T) {
+	// read from filesystem
+	m, err := NewFromFS(&ManifestScanOptions{
+		IgnoreDotfiles: true,
+	}, big)
+	assert.Nil(t, err)
+	assert.NotNil(t, m)
+
+	// scan for files
+	m.Scan()
+
+	// serialize
+	var buf bytes.Buffer
+	m.WriteTo(&buf)
+
+	// show serialized
+	log.Dump(buf.Bytes())
+
+	// do it again
+	var buf2 bytes.Buffer
+	m.WriteTo(&buf2)
+
+	// should be same!
+	assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
+
+	// deserialize
+	m2, err := NewFromProto(&buf)
+	assert.Nil(t, err)
+	assert.NotNil(t, m2)
+
+	log.Dump(m2)
+}

mfer/manifest.go

@@ -65,8 +65,7 @@ func (m *manifest) addInputPath(inputPath string) error {
 	}
 	// FIXME check to make sure inputPath/abs exists maybe
 	afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs))
-	m.addInputFS(afs)
+	return m.addInputFS(afs)
-	return nil
 }
 
 func (m *manifest) addInputFS(f afero.Fs) error {
@@ -159,7 +158,6 @@ func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
 
 func (m *manifest) Scan() error {
 	// FIXME scan and whatever function does the hashing should take ctx
-	log.Debug("manifest scanning")
 	for idx, sfs := range m.sourceFS {
 		if sfs == nil {
 			return errors.New("invalid source fs")

mfer/mf.proto

@@ -9,19 +9,26 @@ message Timestamp {
 
 message MFFileOuter {
     enum Version {
-        NONE = 0;
+        VERSION_NONE = 0;
-        ONE = 1; // only one for now
+        VERSION_ONE = 1; // only one for now
     }
 
     // required mffile root attributes 1xx
     Version version = 101;
-    bytes innerMessage = 102;
+
+    enum CompressionType {
+        COMPRESSION_NONE = 0;
+        COMPRESSION_GZIP = 1;
+    }
+
+    CompressionType compressionType = 102;
+
     // these are used solely to detect corruption/truncation
     // and not for cryptographic integrity.
     int64 size = 103;
     bytes sha256 = 104;
-    bytes deleteme = 105;
 
+    bytes innerMessage = 199;
     // 2xx for optional manifest root attributes
     // think we might use gosignify instead of gpg:
     // github.com/frankbraun/gosignify
@@ -57,8 +64,8 @@ message MFFileChecksum {
 
 message MFFile {
     enum Version {
-        NONE = 0;
+        VERSION_NONE = 0;
-        ONE = 1; // only one for now
+        VERSION_ONE = 1; // only one for now
     }
     Version version = 100;
 

mfer/mfer_test.go

@@ -2,10 +2,10 @@ package mfer
 
 import (
 	"bytes"
+	"fmt"
 	"testing"
 
 	"git.eeqj.de/sneak/mfer/internal/log"
-	"github.com/davecgh/go-spew/spew"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 )
@@ -16,19 +16,25 @@ var (
 )
 
 var (
-	mf afero.Fs     = afero.NewMemMapFs()
+	af  *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
-	af *afero.Afero = &afero.Afero{Fs: mf}
+	big *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
 )
 
 func init() {
+	log.EnableDebugLogging()
+
 	// create test files and directories
 	af.MkdirAll("/a/b/c", 0o755)
 	af.MkdirAll("/.hidden", 0o755)
-	afero.WriteFile(af, "/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
+	af.WriteFile("/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
-	afero.WriteFile(af, "/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
+	af.WriteFile("/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
-	afero.WriteFile(af, "/.hidden/hello.txt", []byte("hello world\n"), 0o755)
+	af.WriteFile("/.hidden/hello.txt", []byte("hello world\n"), 0o755)
-	afero.WriteFile(af, "/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
+	af.WriteFile("/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
-	log.EnableDebugLogging()
+
+	big.MkdirAll("/home/user/Library", 0o755)
+	for i := range [25]int{} {
+		big.WriteFile(fmt.Sprintf("/home/user/Library/hello%d.txt", i), []byte("hello world\n"), 0o755)
+	}
 }
 
 func TestPathHiddenFunc(t *testing.T) {
@@ -45,6 +51,7 @@ func TestManifestGenerationOne(t *testing.T) {
 	}, af)
 	assert.Nil(t, err)
 	assert.NotNil(t, m)
+	m.Scan()
 	assert.Equal(t, int64(2), m.GetFileCount())
 	assert.Equal(t, int64(30), m.GetTotalFileSize())
 }
@@ -55,7 +62,7 @@ func TestManifestGenerationTwo(t *testing.T) {
 	}, af)
 	assert.Nil(t, err)
 	assert.NotNil(t, m)
-	spew.Dump(m)
+	m.Scan()
 	assert.Equal(t, int64(4), m.GetFileCount())
 	assert.Equal(t, int64(54), m.GetTotalFileSize())
 	err = m.generate()
@@ -63,5 +70,5 @@ func TestManifestGenerationTwo(t *testing.T) {
 	var buf bytes.Buffer
 	err = m.WriteTo(&buf)
 	assert.Nil(t, err)
-	spew.Dump(buf)
+	log.Dump(buf.Bytes())
 }

mfer/serialize.go

@@ -2,15 +2,19 @@ package mfer
 
 import (
 	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
 	"errors"
 	"time"
 
-	"git.eeqj.de/sneak/mfer/internal/log"
 	"google.golang.org/protobuf/proto"
 )
 
 //go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
 
+// rot13("MANIFEST")
+const MAGIC string = "ZNAVSRFG"
+
 func newTimestampFromTime(t time.Time) *Timestamp {
 	out := &Timestamp{
 		Seconds: t.Unix(),
@@ -20,10 +24,6 @@ func newTimestampFromTime(t time.Time) *Timestamp {
 }
 
 func (m *manifest) generate() error {
-	log.Debug("generate()")
-
-	const MAGIC string = "ZNAVSRFG"
-
 	if m.pbInner == nil {
 		e := m.generateInner()
 		if e != nil {
@@ -36,7 +36,7 @@ func (m *manifest) generate() error {
 			return e
 		}
 	}
-	dat, err := proto.Marshal(m.pbOuter)
+	dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
 	if err != nil {
 		return err
 	}
@@ -49,29 +49,46 @@ func (m *manifest) generate() error {
 }
 
 func (m *manifest) generateOuter() error {
-	log.Debug("generateOuter()")
 	if m.pbInner == nil {
 		return errors.New("internal error")
 	}
-	innerData, err := proto.Marshal(m.pbInner)
+	innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
 	if err != nil {
 		return err
 	}
+
+	h := sha256.New()
+	h.Write(innerData)
+
+	idc := new(bytes.Buffer)
+	gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
+	if err != nil {
+		return err
+	}
+	_, err = gzw.Write(innerData)
+	if err != nil {
+		return err
+	}
+
+	gzw.Close()
+
 	o := &MFFileOuter{
-		InnerMessage: innerData,
+		InnerMessage:    idc.Bytes(),
+		Size:            int64(len(innerData)),
+		Sha256:          h.Sum(nil),
+		Version:         MFFileOuter_VERSION_ONE,
+		CompressionType: MFFileOuter_COMPRESSION_GZIP,
 	}
 	m.pbOuter = o
 	return nil
 }
 
 func (m *manifest) generateInner() error {
-	log.Debug("generateInner()")
 	m.pbInner = &MFFile{
-		Version:   MFFile_ONE,
+		Version:   MFFile_VERSION_ONE,
 		CreatedAt: newTimestampFromTime(time.Now()),
 		Files:     []*MFFilePath{},
 	}
-
 	for _, f := range m.files {
 		nf := &MFFilePath{
 			Path: f.path,
@@ -79,6 +96,5 @@ func (m *manifest) generateInner() error {
 		}
 		m.pbInner.Files = append(m.pbInner.Files, nf)
 	}
-	log.Dump(m.pbInner)
 	return nil
 }