add link to styleguide

update readme to conform with my new readme howto standards (#8 )
Reviewed-on: #8 manually approving because CI is offline rn for some reason
2024-12-09 02:56:18 +00:00 · 2024-12-09 02:50:57 +00:00 · 2022-12-09 00:02:33 +00:00
28 changed files with 773 additions and 281 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,5 +1,3 @@
-*.tzst
-*.tar
-/buildimage
-/dockerdeps
-/tmp
+*.tmp
+*.dockerimage
+.git
--- a/.drone.yml
+++ b/.drone.yml
@@ -7,8 +7,17 @@ steps:
  network_mode: bridge
  settings:
    repo: sneak/mfer
+    build_args_from_env: [ DRONE_COMMIT_SHA ]
    dry_run: true
    custom_dns: [ 116.202.204.30 ]
    tags:
-      - ${DRONE_COMMIT_SHA}
+      - ${DRONE_COMMIT_SHA:0:7}
      - ${DRONE_BRANCH}
+      - latest
+- name: notify
+  image: plugins/slack
+  settings:
+    webhook:
+      from_secret: SLACK_WEBHOOK_URL
+  when:
+    event: pull_request
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,6 @@
 mfer/*.pb.go
 /mfer.cmd
-vendor
 /tmp
-/buildimage/go.mod
-/buildimage/go.sum
+*.tmp
+*.dockerimage
+/vendor
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -0,0 +1,2 @@
+run:
+    tests: false
--- a/56
+++ b/56
@@ -1,29 +1,37 @@
-## lint image
-FROM golangci/golangci-lint:v1.50.1
-
-RUN mkdir -p /build
+################################################################################
+#2345678911234567892123456789312345678941234567895123456789612345678971234567898
+################################################################################
+FROM sneak/builder:2022-12-08 AS builder
+ENV DEBIAN_FRONTEND noninteractive
 WORKDIR /build
-COPY ./ ./
-RUN golangci-lint run
-
-## build image:
-FROM sneak/mfer-build AS builder
-WORKDIR /build
-
-COPY go.mod .
-COPY go.sum .
-
-COPY ./ ./
-# don't lint again during build because there's no golangci-lint in this
-# image and we already did it in a previous stage
-#RUN make lint
-RUN make mfer.cmd
-#RUN go mod vendor
-RUN tar -c . | bzip2 > /src.tbz2
-
+COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum /build/
+COPY ./vendor.tzst /build/vendor.tzst
+COPY ./modcache.tzst /build/modcache.tzst
+COPY ./internal ./internal
+COPY ./bin/gitrev.sh ./bin/gitrev.sh
+COPY ./mfer ./mfer
+COPY ./cmd ./cmd
+ARG GITREV unknown
+ARG DRONE_COMMIT_SHA unknown

+RUN mkdir -p "$(go env GOMODCACHE)" && cd "$(go env GOMODCACHE)" && \
+    zstdmt -d --stdout /build/modcache.tzst | tar xf - && \
+    rm /build/modcache.tzst && cd /build
+RUN \
+    cd mfer && go generate . && cd .. && \
+    GOPACKAGESDEBUG=true golangci-lint run ./... && \
+    mkdir vendor && cd vendor && \
+    zstdmt -d --stdout /build/vendor.tzst | tar xf - && rm /build/vendor.tzst && \
+    cd .. && \
+    make mfer.cmd
+RUN rm -rf /build/vendor && go mod vendor && tar -c . | zstdmt -19 > /src.tzst
+################################################################################
+#2345678911234567892123456789312345678941234567895123456789612345678971234567898
+################################################################################
+## final image
+################################################################################
 FROM scratch
-COPY --from=builder /src.tbz2 /src.tbz2
+# we put all the source into the final image for posterity, it's small
+COPY --from=builder /src.tzst /src.tzst
 COPY --from=builder /build/mfer.cmd /mfer
 ENTRYPOINT ["/mfer"]
-
--- a/57
+++ b/57
@@ -1,46 +1,52 @@
 export DOCKER_BUILDKIT := 1
+export PROGRESS_NO_TRUNC := 1
 GOPATH := $(shell go env GOPATH)
 export PATH := $(PATH):$(GOPATH)/bin
 PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
-
+SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
 ARCH := $(shell uname -m)
-GITREV := $(shell git describe --always --dirty=-dirty)
+GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh)
 APPNAME := mfer
+VERSION := 0.1.0
 export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
-
-GOLDFLAGS += -X main.Version=0.1.0
-GOLDFLAGS += -X main.Gitrev=$(GITREV)
+GOLDFLAGS += -X main.Version=$(VERSION)
+GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
 GOFLAGS := -ldflags "$(GOLDFLAGS)"

-default: test
+.PHONY: docker default run ci test fixme
+
+default: fmt test

 run: ./mfer.cmd
 	./$<
 	./$< gen --ignore-dotfiles

-test: fmt
+ci: test
+
+test: $(SOURCEFILES) mfer/mf.pb.go
 	go test -v --timeout 3s ./...

 $(PROTOC_GEN_GO):
-	test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@latest
+	test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1

 fixme:
 	@grep -nir fixme . | grep -v Makefile

 devprereqs:
 	which gofumpt || go install -v mvdan.cc/gofumpt@latest
-	which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest
+	which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1

-mfer.cmd: mfer/*.go internal/*/*.go cmd/*/*.go
-	protoc --version
+mfer/mf.pb.go: mfer/mf.proto
 	cd mfer && go generate .
-	make test
+
+mfer.cmd: $(SOURCEFILES) mfer/mf.pb.go
+	protoc --version
 	cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../mfer.cmd $(GOFLAGS) .

 clean:
-	rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer
+	rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer *.dockerimage

-fmt: devprereqs
+fmt: mfer/mf.pb.go
 	gofumpt -l -w mfer internal cmd
 	golangci-lint run --fix
 	-prettier -w *.json
@@ -50,10 +56,27 @@ lint:
 	golangci-lint run
 	sh -c 'test -z "$$(gofmt -l .)"'

-docker:
-	bash -x bin/docker-prereqs.sh
-	docker build .
+docker: sneak-mfer.$(ARCH).tzst.dockerimage
+
+sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
+	docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
+	docker save sneak/mfer | pv | zstdmt -19 > $@
+	du -sh $@

 godoc:
 	open http://127.0.0.1:6060
 	godoc -http=:6060
+
+vendor.tzst: go.mod go.sum
+	go mod tidy
+	go mod vendor
+	cd vendor && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
+	rm -rf vendor
+	mv $@.tmp $@
+
+modcache.tzst: go.mod go.sum
+	go mod tidy
+	cd $(HOME)/go/pkg && chmod -R u+rw . && rm -rf mod sumdb
+	go mod download -x
+	cd $(shell go env GOMODCACHE) && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
+	mv $@.tmp $@
--- a/README.md
+++ b/README.md
@@ -1,11 +1,50 @@
 # mfer

-Manifest file generator and checker.
+[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library
+and thin wrapper command-line utility written in [Go](https://golang.org)
+and first published in 2022 under the [WTFPL](https://wtfpl.net) (public
+domain) license.  It specifies and generates `.mf` manifest files over a
+directory tree of files to encapsulate metadata about them (such as
+cryptographic checksums or signatures over same) to aid in archiving,
+downloading, and streaming, or mirroring.  The manifest files' data is
+serialized with Google's [protobuf serialization
+format](https://developers.google.com/protocol-buffers).  The structure of
+these files can be found [in the format
+specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
+which is included in the [project
+repository](https://git.eeqj.de/sneak/mfer).
+
+The current version is pre-1.0 and while the repo was published in 2022,
+there has not yet been any versioned release.  [SemVer](https://semver.org)
+will be used for releases.
+
+This project was started by [@sneak](https://sneak.berlin) to scratch an
+itch in 2022 and is currently a one-person effort, though the goal is for
+this to emerge as a de-facto standard and be incorporated into other
+software.  A compatible javascript library is planned.

 # Build Status

 [![Build Status](https://drone.datavi.be/api/badges/sneak/mfer/status.svg)](https://drone.datavi.be/sneak/mfer)

+# Participation
+
+The community is as yet nonexistent so there are no defined policies or
+norms yet.  Primary development happens on a privately-run Gitea instance at
+[https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) and issues
+are [tracked there](https://git.eeqj.de/sneak/mfer/issues).
+
+Changes must always be formatted with a standard `go fmt`, syntactically
+valid, and must pass the linting defined in the repository (presently only
+the `golangci-lint` defaults), which can be run with a `make lint`.  The
+`main` branch is protected and all changes must be made via [pull
+requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
+Any changes submitted to this project must also be
+[WTFPL-licensed](https://wtfpl.net) to be considered.
+
+Ideally, contributions conform to @sneak's personal 
+[code styleguide](https://git.eeqj.de/sneak/styleguide).
+
 # Problem Statement

 Given a plain URL, there is no standard way to safely and programmatically
@@ -170,6 +209,15 @@ regardless of filesystem format.
 Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your
 desired username for an account on this Gitea instance.

-I am currently interested in hiring a contractor skilled with the Go
-standard library interfaces to specify this tool in full and develop a
-prototype implementation.
+## Links
+
+* Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
+* Issues: [https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
+
+# Authors
+
+* [@sneak &lt;sneak@sneak.berlin&gt;](mailto:sneak@sneak.berlin)
+
+# License
+
+* [WTFPL](https://wtfpl.net)
--- a/bin/docker-prereqs.sh
+++ b/bin/docker-prereqs.sh
@@ -1,50 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-GOI="golang:1.19.3-bullseye"
-CII="golangci/golangci-lint:v1.50.1"
-BII="sneak/mfer-build"
-
-if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then
-    mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
-fi
-
-DICD="$DOCKER_IMAGE_CACHE_DIR"
-
-function buildImageCache() {
-    if [[ ! -e "$DICD/go.tzst" ]]; then
-        docker pull $GOI
-        docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \
-            mv $DICD/go.tzst.tmp $DICD/go.tzst
-    fi
-
-    if [[ ! -e "$DICD/ci.tzst" ]]; then
-        docker pull $CII
-        docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \
-            mv $DICD/ci.tzst.tmp $DICD/ci.tzst
-    fi
-
-    if [[ ! -e "$DICD/build.tzst" ]]; then
-        go mod download -x
-        cd buildimage
-        cp ../go.mod ../go.sum .
-        docker build -t $BII . && rm -rf go.mod go.sum && \
-        cd .. && \
-        docker save $BII | pv | zstdmt -19 > $DICD/build.tzst.tmp && \
-            mv $DICD/build.tzst.tmp $DICD/build.tzst
-    fi
-}
-
-function loadImageCache() {
-    docker image ls $CII || \
-        zstdmt -d --stdout $DICD/ci.tzst | pv | docker load
-
-    docker image ls $GOI || \
-        zstdmt -d --stdout $DICD/go.tzst | pv | docker load
-
-    docker image ls $BII || \
-        zstdmt -d --stdout $DICD/build.tzst | pv | docker load
-}
-
-buildImageCache
-
-loadImageCache
--- a/bin/gitrev.sh
+++ b/bin/gitrev.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+if [[ ! -z "$DRONE_COMMIT_SHA" ]]; then
+    echo "${DRONE_COMMIT_SHA:0:7}"
+    exit 0
+fi
+
+if [[ ! -z "$GITREV" ]]; then
+    echo $GITREV
+else
+    git describe --always --dirty=-dirty
+fi
--- a/buildimage/Dockerfile
+++ b/buildimage/Dockerfile
@@ -1,21 +0,0 @@
-## build image:
-FROM golang:1.19.3-bullseye AS builder
-
-ENV DEBIAN_FRONTEND noninteractive
-RUN apt update && apt install -y make bzip2 curl unzip
-RUN mkdir -p /build
-WORKDIR /build
-
-# install newer protoc
-RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \
-    unzip *.zip -d /usr/local && rm -v *.zip && protoc --version
-
-RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
-
-RUN go env
-
-COPY ./go.mod .
-COPY ./go.sum .
-
-RUN --mount=type=cache,target=/go/pkg go mod download -x
-RUN rm -rfv /var/cache/* /var/tmp/*
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module git.eeqj.de/sneak/mfer

-go 1.16
+go 1.17

 require (
 	github.com/apex/log v1.9.0
@@ -8,6 +8,27 @@ require (
 	github.com/pterm/pterm v0.12.35
 	github.com/spf13/afero v1.8.0
 	github.com/stretchr/testify v1.8.1
-	github.com/urfave/cli/v2 v2.3.0
+	github.com/urfave/cli/v2 v2.23.6
 	google.golang.org/protobuf v1.28.1
+
+)
+
+require (
+	github.com/atomicgo/cursor v0.0.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/fatih/color v1.7.0 // indirect
+	github.com/gookit/color v1.4.2 // indirect
+	github.com/mattn/go-colorable v0.1.2 // indirect
+	github.com/mattn/go-isatty v0.0.8 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.4 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -37,6 +37,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
 github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
@@ -61,8 +62,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -188,11 +189,9 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
@@ -217,10 +216,12 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
 github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
 github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
-github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
-github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
+github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
+github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -529,7 +530,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/internal/bork/error.go
+++ b/internal/bork/error.go
@@ -0,0 +1,15 @@
+package bork
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	ErrMissingMagic  = errors.New("missing magic bytes in file")
+	ErrFileTruncated = errors.New("file/stream is truncated abnormally")
+)
+
+func Newf(format string, args ...interface{}) error {
+	return fmt.Errorf(format, args...)
+}
--- a/internal/bork/error_test.go
+++ b/internal/bork/error_test.go
@@ -0,0 +1,11 @@
+package bork
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuild(t *testing.T) {
+	assert.NotNil(t, ErrMissingMagic)
+}
--- a/internal/cli/fetch.go
+++ b/internal/cli/fetch.go
@@ -7,5 +7,6 @@ import (

 func (mfa *CLIApp) fetchManifestOperation(c *cli.Context) error {
 	log.Debugf("fetchManifestOperation()")
-	return nil
+	panic("not implemented")
+	return nil //nolint
 }
--- a/internal/cli/gen.go
+++ b/internal/cli/gen.go
@@ -1,36 +1,54 @@
 package cli

 import (
-	"fmt"
+	"bytes"
+	"path/filepath"

+	"git.eeqj.de/sneak/mfer/internal/log"
 	"git.eeqj.de/sneak/mfer/mfer"
-	"github.com/davecgh/go-spew/spew"
 	"github.com/urfave/cli/v2"
 )

-func (mfa *CLIApp) generateManifestOperation(c *cli.Context) error {
-	fmt.Println("generateManifestOperation()")
-	myArgs := c.Args()
-	spew.Dump(myArgs)
-
-	fmt.Printf("%#v\n", c.Args().First())
-	if c.Args().Len() > 0 {
-		fmt.Printf("%#v\n", c.Args().Get(1))
-	}
-
-	// fmt.Printf("called with arg: %s\n", c.String("input"))
+func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
+	log.Debug("generateManifestOperation()")
+	myArgs := ctx.Args()
+	log.Dump(myArgs)

 	opts := &mfer.ManifestScanOptions{
-		IgnoreDotfiles: c.Bool("IgnoreDotfiles"),
-		FollowSymLinks: c.Bool("FollowSymLinks"),
+		IgnoreDotfiles: ctx.Bool("IgnoreDotfiles"),
+		FollowSymLinks: ctx.Bool("FollowSymLinks"),
 	}
-	// FIXME add command flags for ignoring dotfiles and following symlinks
-	mf, err := mfer.NewFromPath(c.String("input"), opts)
+	paths := make([]string, ctx.Args().Len()-1)
+	for i := 0; i < ctx.Args().Len(); i++ {
+		ap, err := filepath.Abs(ctx.Args().Get(i))
+		if err != nil {
+			return err
+		}
+		log.Dump(ap)
+		paths = append(paths, ap)
+	}
+	mf, err := mfer.NewFromPaths(opts, paths...)
 	if err != nil {
 		panic(err)
 	}
+	mf.WithContext(ctx.Context)

-	spew.Dump(mf)
+	log.Dump(mf)

+	err = mf.Scan()
+	if err != nil {
+		return err
+	}
+
+	buf := new(bytes.Buffer)
+
+	err = mf.WriteTo(buf)
+	if err != nil {
+		return err
+	}
+
+	dat := buf.Bytes()
+
+	log.Dump(dat)
 	return nil
 }
--- a/internal/cli/mfer.go
+++ b/internal/cli/mfer.go
@@ -5,11 +5,8 @@ import (
 	"os"
 	"time"

-	"github.com/pterm/pterm"
+	"git.eeqj.de/sneak/mfer/internal/log"
 	"github.com/urfave/cli/v2"
-
-	"github.com/apex/log"
-	acli "github.com/apex/log/handlers/cli"
 )

 type CLIApp struct {
@@ -21,36 +18,46 @@ type CLIApp struct {
 	app         *cli.App
 }

-func (mfa *CLIApp) printBanner() {
-	s, _ := pterm.DefaultBigText.WithLetters(pterm.NewLettersFromString(mfa.appname)).Srender()
-	pterm.DefaultCenter.Println(s) // Print BigLetters with the default CenterPrinter
-}
+const banner = `              ___           ___         ___           ___
+	     /__/\         /  /\       /  /\         /  /\
+	    |  |::\       /  /:/_     /  /:/_       /  /::\
+	    |  |:|:\     /  /:/ /\   /  /:/ /\     /  /:/\:\
+	  __|__|:|\:\   /  /:/ /:/  /  /:/ /:/_   /  /:/~/:/
+	 /__/::::| \:\ /__/:/ /:/  /__/:/ /:/ /\ /__/:/ /:/___
+	 \  \:\~~\__\/ \  \:\/:/   \  \:\/:/ /:/ \  \:\/:::::/
+	  \  \:\        \  \::/     \  \::/ /:/   \  \::/~~~~
+	   \  \:\        \  \:\      \  \:\/:/     \  \:\
+	    \  \:\        \  \:\      \  \::/       \  \:\
+	     \__\/         \__\/       \__\/         \__\/`

-func (mfa *CLIApp) disableStyling() {
-	pterm.DisableColor()
-	pterm.DisableStyling()
-	pterm.Debug.Prefix.Text = ""
-	pterm.Info.Prefix.Text = ""
-	pterm.Success.Prefix.Text = ""
-	pterm.Warning.Prefix.Text = ""
-	pterm.Error.Prefix.Text = ""
-	pterm.Fatal.Prefix.Text = ""
+func (mfa *CLIApp) printBanner() {
+	fmt.Println(banner)
 }

 func (mfa *CLIApp) VersionString() string {
 	return fmt.Sprintf("%s (%s)", mfa.version, mfa.gitrev)
 }

+func (mfa *CLIApp) setVerbosity(v int) {
+	_, present := os.LookupEnv("MFER_DEBUG")
+	if present {
+		log.EnableDebugLogging()
+	} else {
+		log.SetLevelFromVerbosity(v)
+	}
+}
+
 func (mfa *CLIApp) run() {
 	mfa.startupTime = time.Now()

 	if NO_COLOR {
 		// shoutout to rob pike who thinks it's juvenile
-		mfa.disableStyling()
+		log.DisableStyling()
 	}

-	log.SetHandler(acli.Default)
-	log.SetLevel(log.InfoLevel)
+	log.Init()
+
+	var verbosity int

 	mfa.app = &cli.App{
 		Name:                 mfa.appname,
@@ -62,6 +69,7 @@ func (mfa *CLIApp) run() {
 				Name:    "verbose",
 				Usage:   "Verbosity level",
 				Aliases: []string{"v"},
+				Count:   &verbosity,
 			},
 			&cli.BoolFlag{
 				Name:    "quiet",
@@ -69,12 +77,6 @@ func (mfa *CLIApp) run() {
 				Aliases: []string{"q"},
 			},
 		},
-		Action: func(c *cli.Context) error {
-			if c.Bool("verbose") {
-				log.SetLevel(log.DebugLevel)
-			}
-			return nil
-		},
 		Commands: []*cli.Command{
 			{
 				Name:    "generate",
@@ -84,6 +86,7 @@ func (mfa *CLIApp) run() {
 					if !c.Bool("quiet") {
 						mfa.printBanner()
 					}
+					mfa.setVerbosity(verbosity)
 					return mfa.generateManifestOperation(c)
 				},
 				Flags: []cli.Flag{
@@ -97,13 +100,6 @@ func (mfa *CLIApp) run() {
 						Aliases: []string{"ignore-dotfiles"},
 						Usage:   "Ignore any dot (hidden) files encountered",
 					},
-					// FIXME this should be a positional arg
-					&cli.StringFlag{
-						Name:    "input",
-						Value:   ".",
-						Aliases: []string{"i"},
-						Usage:   "Specify input directory.",
-					},
 					&cli.StringFlag{
 						Name:    "output",
 						Value:   "./index.mf",
@@ -119,6 +115,7 @@ func (mfa *CLIApp) run() {
 					if !c.Bool("quiet") {
 						mfa.printBanner()
 					}
+					mfa.setVerbosity(verbosity)
 					return mfa.checkManifestOperation(c)
 				},
 			},
@@ -137,6 +134,7 @@ func (mfa *CLIApp) run() {
 					if !c.Bool("quiet") {
 						mfa.printBanner()
 					}
+					mfa.setVerbosity(verbosity)
 					return mfa.fetchManifestOperation(c)
 				},
 			},
--- a/internal/log/log.go
+++ b/internal/log/log.go
@@ -0,0 +1,89 @@
+package log
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/apex/log"
+	acli "github.com/apex/log/handlers/cli"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/pterm/pterm"
+)
+
+type Level = log.Level
+
+func DisableStyling() {
+	pterm.DisableColor()
+	pterm.DisableStyling()
+	pterm.Debug.Prefix.Text = ""
+	pterm.Info.Prefix.Text = ""
+	pterm.Success.Prefix.Text = ""
+	pterm.Warning.Prefix.Text = ""
+	pterm.Error.Prefix.Text = ""
+	pterm.Fatal.Prefix.Text = ""
+}
+
+func Init() {
+	log.SetHandler(acli.Default)
+	log.SetLevel(log.InfoLevel)
+}
+
+func Debugf(format string, args ...interface{}) {
+	DebugReal(fmt.Sprintf(format, args...), 2)
+}
+
+func Debug(arg string) {
+	DebugReal(arg, 2)
+}
+
+func DebugReal(arg string, cs int) {
+	_, callerFile, callerLine, ok := runtime.Caller(cs)
+	if !ok {
+		return
+	}
+	tag := fmt.Sprintf("%s:%d: ", callerFile, callerLine)
+	log.Debug(tag + arg)
+}
+
+func Dump(args ...interface{}) {
+	DebugReal(spew.Sdump(args...), 2)
+}
+
+func EnableDebugLogging() {
+	SetLevel(log.DebugLevel)
+}
+
+func VerbosityStepsToLogLevel(l int) log.Level {
+	switch l {
+	case 1:
+		return log.WarnLevel
+	case 2:
+		return log.InfoLevel
+	case 3:
+		return log.DebugLevel
+	}
+	return log.ErrorLevel
+}
+
+func SetLevelFromVerbosity(l int) {
+	SetLevel(VerbosityStepsToLogLevel(l))
+}
+
+func SetLevel(arg log.Level) {
+	log.SetLevel(arg)
+}
+
+func GetLogger() *log.Logger {
+	if logger, ok := log.Log.(*log.Logger); ok {
+		return logger
+	}
+	panic("unable to get logger")
+}
+
+func GetLevel() log.Level {
+	return GetLogger().Level
+}
+
+func WithError(e error) *log.Entry {
+	return GetLogger().WithError(e)
+}
--- a/internal/log/log_test.go
+++ b/internal/log/log_test.go
@@ -0,0 +1,12 @@
+package log
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuild(t *testing.T) {
+	Init()
+	assert.True(t, true)
+}
--- a/mfer/deserialize.go
+++ b/mfer/deserialize.go
@@ -0,0 +1,89 @@
+package mfer
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"io"
+
+	"git.eeqj.de/sneak/mfer/internal/bork"
+	"git.eeqj.de/sneak/mfer/internal/log"
+	"google.golang.org/protobuf/proto"
+)
+
+func (m *manifest) validateProtoOuter() error {
+	if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
+		return errors.New("unknown version")
+	}
+	if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
+		return errors.New("unknown compression type")
+	}
+
+	bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
+
+	gzr, err := gzip.NewReader(bb)
+	if err != nil {
+		return err
+	}
+
+	dat, err := io.ReadAll(gzr)
+	defer gzr.Close()
+
+	if err != nil {
+		return err
+	}
+
+	isize := len(dat)
+	if int64(isize) != m.pbOuter.Size {
+		log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
+		return bork.ErrFileTruncated
+	}
+	log.Debugf("inner data size is %d", isize)
+	log.Dump(dat)
+	log.Dump(m.pbOuter.Sha256)
+	return nil
+}
+
+func validateMagic(dat []byte) bool {
+	ml := len([]byte(MAGIC))
+	if len(dat) < ml {
+		return false
+	}
+	got := dat[0:ml]
+	expected := []byte(MAGIC)
+	return bytes.Equal(got, expected)
+}
+
+func NewFromProto(input io.Reader) (*manifest, error) {
+	m := New()
+	dat, err := io.ReadAll(input)
+	if err != nil {
+		return nil, err
+	}
+	if !validateMagic(dat) {
+		return nil, errors.New("invalid file format")
+	}
+
+	// remove magic bytes prefix:
+	ml := len([]byte(MAGIC))
+	bb := bytes.NewBuffer(dat[ml:])
+	dat = bb.Bytes()
+
+	log.Dump(dat)
+
+	// deserialize:
+	m.pbOuter = new(MFFileOuter)
+	err = proto.Unmarshal(dat, m.pbOuter)
+
+	if err != nil {
+		return nil, err
+	}
+
+	ve := m.validateProtoOuter()
+	if ve != nil {
+		return nil, ve
+	}
+
+	// FIXME TODO deserialize inner
+	return m, nil
+}
--- a/mfer/example_test.go
+++ b/mfer/example_test.go
@@ -0,0 +1,42 @@
+package mfer
+
+import (
+	"bytes"
+	"testing"
+
+	"git.eeqj.de/sneak/mfer/internal/log"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAPIExample(t *testing.T) {
+	// read from filesystem
+	m, err := NewFromFS(&ManifestScanOptions{
+		IgnoreDotfiles: true,
+	}, big)
+	assert.Nil(t, err)
+	assert.NotNil(t, m)
+
+	// scan for files
+	m.Scan()
+
+	// serialize
+	var buf bytes.Buffer
+	m.WriteTo(&buf)
+
+	// show serialized
+	log.Dump(buf.Bytes())
+
+	// do it again
+	var buf2 bytes.Buffer
+	m.WriteTo(&buf2)
+
+	// should be same!
+	assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
+
+	// deserialize
+	m2, err := NewFromProto(&buf)
+	assert.Nil(t, err)
+	assert.NotNil(t, m2)
+
+	log.Dump(m2)
+}
--- a/mfer/manifest.go
+++ b/mfer/manifest.go
@@ -1,36 +1,42 @@
 package mfer

 import (
+	"bytes"
+	"context"
+	"errors"
 	"fmt"
 	"io/fs"
 	"path"
 	"path/filepath"
 	"strings"

+	"git.eeqj.de/sneak/mfer/internal/log"
 	"github.com/spf13/afero"
 )

-type ManifestFile struct {
-	Path     string
-	FileInfo fs.FileInfo
+type manifestFile struct {
+	path string
+	info fs.FileInfo
 }

-func (m *ManifestFile) String() string {
-	return fmt.Sprintf("<File \"%s\">", m.Path)
+func (m *manifestFile) String() string {
+	return fmt.Sprintf("<File \"%s\">", m.path)
 }

-type Manifest struct {
-	SourceFS      afero.Fs
-	SourceFSRoot  string
-	Files         []*ManifestFile
-	ScanOptions   *ManifestScanOptions
-	TotalFileSize int64
-	PBInner       *MFFile
-	PBOuter       *MFFileOuter
+type manifest struct {
+	sourceFS      []afero.Fs
+	files         []*manifestFile
+	scanOptions   *ManifestScanOptions
+	totalFileSize int64
+	pbInner       *MFFile
+	pbOuter       *MFFileOuter
+	output        *bytes.Buffer
+	ctx           context.Context
+	errors        []*error
 }

-func (m *Manifest) String() string {
-	return fmt.Sprintf("<Manifest count=%d totalSize=%d>", len(m.Files), m.TotalFileSize)
+func (m *manifest) String() string {
+	return fmt.Sprintf("<Manifest count=%d totalSize=%d>", len(m.files), m.totalFileSize)
 }

 type ManifestScanOptions struct {
@@ -38,38 +44,73 @@ type ManifestScanOptions struct {
 	FollowSymLinks bool
 }

-func NewFromPath(inputPath string, options *ManifestScanOptions) (*Manifest, error) {
+func (m *manifest) HasError() bool {
+	return len(m.errors) > 0
+}
+
+func (m *manifest) AddError(e error) *manifest {
+	m.errors = append(m.errors, &e)
+	return m
+}
+
+func (m *manifest) WithContext(c context.Context) *manifest {
+	m.ctx = c
+	return m
+}
+
+func (m *manifest) addInputPath(inputPath string) error {
 	abs, err := filepath.Abs(inputPath)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	afs := afero.NewBasePathFs(afero.NewOsFs(), abs)
-	m, err := NewFromFS(afs, options)
+	// FIXME check to make sure inputPath/abs exists maybe
+	afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs))
+	return m.addInputFS(afs)
+}
+
+func (m *manifest) addInputFS(f afero.Fs) error {
+	if m.sourceFS == nil {
+		m.sourceFS = make([]afero.Fs, 0)
+	}
+	m.sourceFS = append(m.sourceFS, f)
+	// FIXME do some sort of check on f here?
+	return nil
+}
+
+func New() *manifest {
+	m := &manifest{}
+	return m
+}
+
+func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) {
+	log.Dump(inputPaths)
+	m := New()
+	m.scanOptions = options
+	for _, p := range inputPaths {
+		err := m.addInputPath(p)
 		if err != nil {
 			return nil, err
 		}
-	m.SourceFSRoot = abs
+	}
 	return m, nil
 }

-func NewFromFS(fs afero.Fs, options *ManifestScanOptions) (*Manifest, error) {
-	m := &Manifest{
-		SourceFS:    fs,
-		ScanOptions: options,
-	}
-	err := m.Scan()
+func NewFromFS(options *ManifestScanOptions, fs afero.Fs) (*manifest, error) {
+	m := New()
+	m.scanOptions = options
+	err := m.addInputFS(fs)
 	if err != nil {
 		return nil, err
 	}
 	return m, nil
 }

-func (m *Manifest) GetFileCount() int64 {
-	return int64(len(m.Files))
+func (m *manifest) GetFileCount() int64 {
+	return int64(len(m.files))
 }

-func (m *Manifest) GetTotalFileSize() int64 {
-	return m.TotalFileSize
+func (m *manifest) GetTotalFileSize() int64 {
+	return m.totalFileSize
 }

 func pathIsHidden(p string) bool {
@@ -89,39 +130,44 @@ func pathIsHidden(p string) bool {
 	}
 }

-/*
-func timeToTimestamp(t time.Time) *Timestamp {
-
-}
-*/
-
-func (m *Manifest) Scan() error {
-	// FIXME scan and whatever function does the hashing should take ctx
-	oe := afero.Walk(m.SourceFS, "/", func(p string, info fs.FileInfo, err error) error {
-		if m.ScanOptions.IgnoreDotfiles && pathIsHidden(p) {
+func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
+	if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) {
 		return nil
 	}
-
-		if info != nil && info.IsDir() {
+	if fi != nil && fi.IsDir() {
 		// manifests contain only files, directories are implied.
 		return nil
 	}
-
-		fileinfo, staterr := m.SourceFS.Stat(p)
+	// FIXME test if 'fi' is already result of stat
+	fileinfo, staterr := m.sourceFS[sfsIndex].Stat(p)
 	if staterr != nil {
-			panic(staterr)
+		return staterr
 	}
-
-		nf := &ManifestFile{
-			Path:     p,
-			FileInfo: fileinfo,
+	cleanPath := p
+	if cleanPath[0:1] == "/" {
+		cleanPath = cleanPath[1:]
 	}
-		m.Files = append(m.Files, nf)
-		m.TotalFileSize = m.TotalFileSize + info.Size()
+	nf := &manifestFile{
+		path: cleanPath,
+		info: fileinfo,
+	}
+	m.files = append(m.files, nf)
+	m.totalFileSize = m.totalFileSize + fi.Size()
 	return nil
+}
+
+func (m *manifest) Scan() error {
+	// FIXME scan and whatever function does the hashing should take ctx
+	for idx, sfs := range m.sourceFS {
+		if sfs == nil {
+			return errors.New("invalid source fs")
+		}
+		e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error {
+			return m.addFile(p, info, idx)
 		})
-	if oe != nil {
-		return oe
+		if e != nil {
+			return e
+		}
 	}
 	return nil
 }
--- a/mfer/mf.proto
+++ b/mfer/mf.proto
@@ -1,6 +1,6 @@
 syntax = "proto3";

-option go_package = "git.eeqj.de/sneak/mfer";
+option go_package = "git.eeqj.de/sneak/mfer/mfer";

 message Timestamp {
  int64 seconds = 1;
@@ -9,18 +9,26 @@ message Timestamp {

 message MFFileOuter {
    enum Version {
-        NONE = 0;
-        ONE = 1; // only one for now
+        VERSION_NONE = 0;
+        VERSION_ONE = 1; // only one for now
    }

    // required mffile root attributes 1xx
    Version version = 101;
-    bytes innerMessage = 102;
+
+    enum CompressionType {
+        COMPRESSION_NONE = 0;
+        COMPRESSION_GZIP = 1;
+    }
+
+    CompressionType compressionType = 102;
+
    // these are used solely to detect corruption/truncation
    // and not for cryptographic integrity.
    int64 size = 103;
    bytes sha256 = 104;

+    bytes innerMessage = 199;
    // 2xx for optional manifest root attributes
    // think we might use gosignify instead of gpg:
    // github.com/frankbraun/gosignify
@@ -56,8 +64,8 @@ message MFFileChecksum {

 message MFFile {
    enum Version {
-        NONE = 0;
-        ONE = 1; // only one for now
+        VERSION_NONE = 0;
+        VERSION_ONE = 1; // only one for now
    }
    Version version = 100;

--- a/mfer/mfer_test.go
+++ b/mfer/mfer_test.go
@@ -1,9 +1,11 @@
 package mfer

 import (
+	"bytes"
+	"fmt"
 	"testing"

-	"github.com/davecgh/go-spew/spew"
+	"git.eeqj.de/sneak/mfer/internal/log"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 )
@@ -14,18 +16,25 @@ var (
 )

 var (
-	mf afero.Fs     = afero.NewMemMapFs()
-	af *afero.Afero = &afero.Afero{Fs: mf}
+	af  *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
+	big *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
 )

 func init() {
+	log.EnableDebugLogging()
+
 	// create test files and directories
 	af.MkdirAll("/a/b/c", 0o755)
 	af.MkdirAll("/.hidden", 0o755)
-	afero.WriteFile(af, "/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
-	afero.WriteFile(af, "/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
-	afero.WriteFile(af, "/.hidden/hello.txt", []byte("hello world\n"), 0o755)
-	afero.WriteFile(af, "/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
+	af.WriteFile("/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
+	af.WriteFile("/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
+	af.WriteFile("/.hidden/hello.txt", []byte("hello world\n"), 0o755)
+	af.WriteFile("/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
+
+	big.MkdirAll("/home/user/Library", 0o755)
+	for i := range [25]int{} {
+		big.WriteFile(fmt.Sprintf("/home/user/Library/hello%d.txt", i), []byte("hello world\n"), 0o755)
+	}
 }

 func TestPathHiddenFunc(t *testing.T) {
@@ -37,22 +46,29 @@ func TestPathHiddenFunc(t *testing.T) {
 }

 func TestManifestGenerationOne(t *testing.T) {
-	m, err := NewFromFS(mf, &ManifestScanOptions{
+	m, err := NewFromFS(&ManifestScanOptions{
 		IgnoreDotfiles: true,
-	})
+	}, af)
 	assert.Nil(t, err)
 	assert.NotNil(t, m)
+	m.Scan()
 	assert.Equal(t, int64(2), m.GetFileCount())
 	assert.Equal(t, int64(30), m.GetTotalFileSize())
 }

 func TestManifestGenerationTwo(t *testing.T) {
-	m, err := NewFromFS(mf, &ManifestScanOptions{
+	m, err := NewFromFS(&ManifestScanOptions{
 		IgnoreDotfiles: false,
-	})
+	}, af)
 	assert.Nil(t, err)
 	assert.NotNil(t, m)
-	spew.Dump(m)
+	m.Scan()
 	assert.Equal(t, int64(4), m.GetFileCount())
 	assert.Equal(t, int64(54), m.GetTotalFileSize())
+	err = m.generate()
+	assert.Nil(t, err)
+	var buf bytes.Buffer
+	err = m.WriteTo(&buf)
+	assert.Nil(t, err)
+	log.Dump(buf.Bytes())
 }
--- a/mfer/output.go
+++ b/mfer/output.go
@@ -5,7 +5,7 @@ import (
 	"os"
 )

-func (m *Manifest) WriteToFile(path string) error {
+func (m *manifest) WriteToFile(path string) error {
 	// FIXME refuse to overwrite without -f if file exists

 	f, err := os.Create(path)
@@ -14,11 +14,20 @@ func (m *Manifest) WriteToFile(path string) error {
 	}
 	defer f.Close()

-	return m.Write(f)
+	return m.WriteTo(f)
 }

-func (m *Manifest) Write(output io.Writer) error {
-	// FIXME implement
-	panic("nope")
-	return nil // nolint:all
+func (m *manifest) WriteTo(output io.Writer) error {
+	if m.pbOuter == nil {
+		err := m.generate()
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err := output.Write(m.output.Bytes())
+	if err != nil {
+		return err
+	}
+	return nil
 }
--- a/mfer/serialize.go
+++ b/mfer/serialize.go
@@ -1,12 +1,100 @@
 package mfer

+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"errors"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+)
+
 //go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto

-func (m *Manifest) Generate() error {
-	m.PBInner = &MFFile{
-		Version: MFFile_ONE,
-		// CreatedAt: time.Now(),
-		Files: []*MFFilePath{},
+// rot13("MANIFEST")
+const MAGIC string = "ZNAVSRFG"
+
+func newTimestampFromTime(t time.Time) *Timestamp {
+	out := &Timestamp{
+		Seconds: t.Unix(),
+		Nanos:   int32(t.UnixNano() - (t.Unix() * 1000000000)),
+	}
+	return out
+}
+
+func (m *manifest) generate() error {
+	if m.pbInner == nil {
+		e := m.generateInner()
+		if e != nil {
+			return e
+		}
+	}
+	if m.pbOuter == nil {
+		e := m.generateOuter()
+		if e != nil {
+			return e
+		}
+	}
+	dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
+	if err != nil {
+		return err
+	}
+	m.output = bytes.NewBuffer([]byte(MAGIC))
+	_, err = m.output.Write(dat)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *manifest) generateOuter() error {
+	if m.pbInner == nil {
+		return errors.New("internal error")
+	}
+	innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
+	if err != nil {
+		return err
+	}
+
+	h := sha256.New()
+	h.Write(innerData)
+
+	idc := new(bytes.Buffer)
+	gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
+	if err != nil {
+		return err
+	}
+	_, err = gzw.Write(innerData)
+	if err != nil {
+		return err
+	}
+
+	gzw.Close()
+
+	o := &MFFileOuter{
+		InnerMessage:    idc.Bytes(),
+		Size:            int64(len(innerData)),
+		Sha256:          h.Sum(nil),
+		Version:         MFFileOuter_VERSION_ONE,
+		CompressionType: MFFileOuter_COMPRESSION_GZIP,
+	}
+	m.pbOuter = o
+	return nil
+}
+
+func (m *manifest) generateInner() error {
+	m.pbInner = &MFFile{
+		Version:   MFFile_VERSION_ONE,
+		CreatedAt: newTimestampFromTime(time.Now()),
+		Files:     []*MFFilePath{},
+	}
+	for _, f := range m.files {
+		nf := &MFFilePath{
+			Path: f.path,
+			// FIXME add more stuff
+		}
+		m.pbInner.Files = append(m.pbInner.Files, nf)
 	}
 	return nil
 }
--- a/modcache.tzst
+++ b/modcache.tzst
--- a/vendor.tzst
+++ b/vendor.tzst
Author	SHA1	Message	Date
Jeffrey Paul	d37f3c141a	add link to styleguide All checks were successful continuous-integration/drone/push Build is passing Details	2024-12-09 02:56:18 +00:00
Jeffrey Paul	2717685619	update readme to conform with my new readme howto standards (#8 ) All checks were successful continuous-integration/drone/push Build is passing Details Reviewed-on: #8 manually approving because CI is offline rn for some reason	2024-12-09 02:50:57 +00:00
Jeffrey Paul	7df558d8d0	next (#5 ) All checks were successful continuous-integration/drone/push Build is passing Details Co-authored-by: sneak <sneak@sneak.berlin> Reviewed-on: #5	2022-12-09 00:02:33 +00:00