small progress

wtf
works but has a weird bug
2022-12-04 03:18:47 +04:00 · 2022-12-02 02:05:09 +01:00 · 2022-12-02 01:54:01 +01:00 · 2022-12-02 01:31:49 +01:00 · 2022-02-02 01:46:51 -08:00 · 2022-02-02 01:46:25 -08:00
35 changed files with 280 additions and 1204 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,3 +0,0 @@
-*.tmp
-*.dockerimage
-.git
--- a/.drone.yml
+++ b/.drone.yml
@@ -7,17 +7,8 @@ steps:
  network_mode: bridge
  settings:
    repo: sneak/mfer
-    build_args_from_env: [ DRONE_COMMIT_SHA ]
    dry_run: true
    custom_dns: [ 116.202.204.30 ]
    tags:
-      - ${DRONE_COMMIT_SHA:0:7}
+      - ${DRONE_COMMIT_SHA}
      - ${DRONE_BRANCH}
-      - latest
- name: notify
-  image: plugins/slack
-  settings:
-    webhook:
-      from_secret: SLACK_WEBHOOK_URL
-  when:
-    event: pull_request
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,2 @@
 mfer/*.pb.go
 /mfer.cmd
-/tmp
-*.tmp
-*.dockerimage
-/vendor
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -1,2 +0,0 @@
-run:
-  tests: false
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -1,2 +0,0 @@
-run:
-    tests: false
--- a/73
+++ b/73
@@ -1,37 +1,42 @@
-################################################################################
-#2345678911234567892123456789312345678941234567895123456789612345678971234567898
-################################################################################
-FROM sneak/builder:2022-12-08 AS builder
-ENV DEBIAN_FRONTEND noninteractive
-WORKDIR /build
-COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum /build/
-COPY ./vendor.tzst /build/vendor.tzst
-COPY ./modcache.tzst /build/modcache.tzst
-COPY ./internal ./internal
-COPY ./bin/gitrev.sh ./bin/gitrev.sh
-COPY ./mfer ./mfer
-COPY ./cmd ./cmd
-ARG GITREV unknown
-ARG DRONE_COMMIT_SHA unknown
+## lint image
+## current as of 2022-01-25
+FROM golangci/golangci-lint@sha256:d16ef91da7e10f3df45c36876543326abbc4c16aaab6548549560b9f52e9e831 AS linter
+
+RUN mkdir -p /build
+WORKDIR /build
+COPY ./ ./
+RUN golangci-lint run
+
+## build image:
+# this is golang:1.17.6-buster as of 2022-01-27
+FROM golang@sha256:52a48e0239f4d645b20ac268a60361703afe7feb2df5697fa89f72052cb87a3e AS builder
+#FROM golang:1.16-buster AS builder
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt update && apt install -y make bzip2 curl unzip
+
+# install newer protoc than what comes with buster
+ENV PB_REL https://github.com/protocolbuffers/protobuf/releases
+RUN curl -LO $PB_REL/download/v3.19.0/protoc-3.19.0-linux-x86_64.zip && \
+    unzip protoc-3.19.0-linux-x86_64.zip -d /usr/local
+
+RUN mkdir -p /build
+WORKDIR /build
+
+COPY go.mod .
+COPY go.sum .
+
+COPY ./ ./
+# don't lint again during build because there's no golangci-lint in this
+# image and we already did it in a previous stage
+#RUN make lint
+RUN make mfer
+#RUN go mod vendor
+RUN tar -c . | bzip2 > /src.tbz2
+

-RUN mkdir -p "$(go env GOMODCACHE)" && cd "$(go env GOMODCACHE)" && \
-    zstdmt -d --stdout /build/modcache.tzst | tar xf - && \
-    rm /build/modcache.tzst && cd /build
-RUN \
-    cd mfer && go generate . && cd .. && \
-    GOPACKAGESDEBUG=true golangci-lint run ./... && \
-    mkdir vendor && cd vendor && \
-    zstdmt -d --stdout /build/vendor.tzst | tar xf - && rm /build/vendor.tzst && \
-    cd .. && \
-    make mfer.cmd
-RUN rm -rf /build/vendor && go mod vendor && tar -c . | zstdmt -19 > /src.tzst
-################################################################################
-#2345678911234567892123456789312345678941234567895123456789612345678971234567898
-################################################################################
-## final image
-################################################################################
 FROM scratch
-# we put all the source into the final image for posterity, it's small
-COPY --from=builder /src.tzst /src.tzst
-COPY --from=builder /build/mfer.cmd /mfer
+COPY --from=builder /src.tbz2 /src.tbz2
+COPY --from=builder /build/mfer /mfer
 ENTRYPOINT ["/mfer"]
+
--- a/71
+++ b/71
@@ -1,82 +1,43 @@
-export DOCKER_BUILDKIT := 1
-export PROGRESS_NO_TRUNC := 1
+
 GOPATH := $(shell go env GOPATH)
 export PATH := $(PATH):$(GOPATH)/bin
 PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
-SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
+
 ARCH := $(shell uname -m)
-GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh)
-APPNAME := mfer
-VERSION := 0.1.0
-export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
+VERSION := $(shell git describe --always --dirty=-dirty)
+
 GOLDFLAGS += -X main.Version=$(VERSION)
-GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
 GOFLAGS := -ldflags "$(GOLDFLAGS)"

-.PHONY: docker default run ci test fixme
-
-default: fmt test
+default: run

 run: ./mfer.cmd
 	./$<
-	./$< gen --ignore-dotfiles
-
-ci: test
-
-test: $(SOURCEFILES) mfer/mf.pb.go
-	go test -v --timeout 3s ./...
+	./$< gen

 $(PROTOC_GEN_GO):
-	test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
-
-fixme:
-	@grep -nir fixme . | grep -v Makefile
+	test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@latest

 devprereqs:
 	which gofumpt || go install -v mvdan.cc/gofumpt@latest
-	which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1
+	which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest

-mfer/mf.pb.go: mfer/mf.proto
-	cd mfer && go generate .
-
-mfer.cmd: $(SOURCEFILES) mfer/mf.pb.go
+mfer.cmd: $(PROTOC_GEN_GO) mfer/*.go internal/*/*.go cmd/*/*.go
 	protoc --version
-	cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../mfer.cmd $(GOFLAGS) .
+	cd mfer && go generate .
+	cd cmd/mfer && go build -o ../../mfer.cmd $(GOFLAGS) .

 clean:
-	rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer *.dockerimage
+	rm -rfv mfer/*.pb.go ./mfer

-fmt: mfer/mf.pb.go
+fmt: prereqs
 	gofumpt -l -w mfer internal cmd
 	golangci-lint run --fix
-	-prettier -w *.json
-	-prettier -w *.md
+	prettier -w *.json *.md

 lint:
 	golangci-lint run
 	sh -c 'test -z "$$(gofmt -l .)"'

-docker: sneak-mfer.$(ARCH).tzst.dockerimage
-
-sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
-	docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
-	docker save sneak/mfer | pv | zstdmt -19 > $@
-	du -sh $@
-
-godoc:
-	open http://127.0.0.1:6060
-	godoc -http=:6060
-
-vendor.tzst: go.mod go.sum
-	go mod tidy
-	go mod vendor
-	cd vendor && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
-	rm -rf vendor
-	mv $@.tmp $@
-
-modcache.tzst: go.mod go.sum
-	go mod tidy
-	cd $(HOME)/go/pkg && chmod -R u+rw . && rm -rf mod sumdb
-	go mod download -x
-	cd $(shell go env GOMODCACHE) && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
-	mv $@.tmp $@
+dockerbuild:
+	docker build .
--- a/README.md
+++ b/README.md
@@ -1,48 +1,11 @@
 # mfer

-[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library
-and thin wrapper command-line utility written in [Go](https://golang.org)
-and first published in 2022 under the [WTFPL](https://wtfpl.net) (public
-domain) license.  It specifies and generates `.mf` manifest files over a
-directory tree of files to encapsulate metadata about them (such as
-cryptographic checksums or signatures over same) to aid in archiving,
-downloading, and streaming, or mirroring.  The manifest files' data is
-serialized with Google's [protobuf serialization
-format](https://developers.google.com/protocol-buffers).  The structure of
-these files can be found [in the format
-specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
-which is included in the [project
-repository](https://git.eeqj.de/sneak/mfer).
-
-The current version is pre-1.0 and while the repo was published in 2022,
-there has not yet been any versioned release.  [SemVer](https://semver.org)
-will be used for releases.
-
-This project was started by [@sneak](https://sneak.berlin) to scratch an
-itch in 2022 and is currently a one-person effort, though the goal is for
-this to emerge as a de-facto standard and be incorporated into other
-software.  A compatible javascript library is planned.
+Manifest file generator and checker.

 # Build Status

 [![Build Status](https://drone.datavi.be/api/badges/sneak/mfer/status.svg)](https://drone.datavi.be/sneak/mfer)

-# Participation
-
-The community is as yet nonexistent so there are no defined policies or
-norms yet.  Primary development happens on a privately-run Gitea instance at
-[https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) and issues
-are [tracked there](https://git.eeqj.de/sneak/mfer/issues).
-
-Changes must always be formatted with a standard `go fmt`, syntactically
-valid, and must pass the linting defined in the repository (presently only
-the `golangci-lint` defaults), which can be run with a `make lint`.  The
-`main` branch is protected and all changes must be made via [pull
-requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
-Any changes submitted to this project must also be
-[WTFPL-licensed](https://wtfpl.net) to be considered.
-
-
 # Problem Statement

 Given a plain URL, there is no standard way to safely and programmatically
@@ -165,7 +128,8 @@ The manifest file would do several important things:
 - a command line option to zero/omit mtime/ctime, as well as manifest
  timestamp, and sort all directory listings so that manifest file
  generation is deterministic/reproducible
- URL format `mfer fetch https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2`
+- URL format `mfer fetch
+  https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2`
  to assert in the URL which PGP signing key should be used in the manifest,
  so that shared URLs have a cryptographic trust root
 - a "well-known" key in the manifest that maps well known keys (could reuse
@@ -207,24 +171,6 @@ regardless of filesystem format.
 Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your
 desired username for an account on this Gitea instance.

-# See Also
-
-## Prior Art: Metalink
-
-* [Metalink - Mozilla Wiki](https://wiki.mozilla.org/Metalink)
-* [Metalink - Wikipedia](https://en.wikipedia.org/wiki/Metalink)
-* [RFC 5854 - The Metalink Download Description Format](https://datatracker.ietf.org/doc/html/rfc5854)
-* [RFC 6249 - Metalink/HTTP: Mirrors and Hashes](https://www.rfc-editor.org/rfc/rfc6249.html)
-
-## Links
-
-* Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
-* Issues: [https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
-
-# Authors
-
-* [@sneak &lt;sneak@sneak.berlin&gt;](mailto:sneak@sneak.berlin)
-
-# License
-
-* [WTFPL](https://wtfpl.net)
+I am currently interested in hiring a contractor skilled with the Go
+standard library interfaces to specify this tool in full and develop a
+prototype implementation.
--- a/TODO.md
+++ b/TODO.md
@@ -1,122 +0,0 @@
-# TODO: mfer 1.0
-
-## Design Questions
-
-*sneak: please answer inline below each question. These are preserved for posterity.*
-
-### Format Design
-
-**1. Should `MFFileChecksum` be simplified?**
-Currently it's a separate message wrapping a single `bytes multiHash` field. Since multihash already self-describes the algorithm, `repeated bytes hashes` directly on `MFFilePath` would be simpler and reduce per-file protobuf overhead. Is the extra message layer intentional (e.g. planning to add per-hash metadata like `verified_at`)?
-
-> *answer:*
-
-**2. Should file permissions/mode be stored?**
-The format stores mtime/ctime but not Unix file permissions. For archival use (ExFAT, filesystem-independent checksums) this may not matter, but for software distribution or filesystem restoration it's a gap. Should we reserve a field now (e.g. `optional uint32 mode = 305`) even if we don't populate it yet?
-
-> *answer:*
-
-**3. Should `atime` be removed from the schema?**
-Access time is volatile, non-deterministic, and often disabled (`noatime`). Including it means two manifests of the same directory at different times will differ, which conflicts with the determinism goal. Remove it, or document it as "never set by default"?
-
-> *answer:*
-
-**4. What are the path normalization rules?**
-The proto has `string path` with no specification about: always forward-slash? Must be relative? No `..` components allowed? UTF-8 NFC vs NFD normalization (macOS vs Linux)? Max path length? This is a security issue (path traversal) and a cross-platform compatibility issue. What rules should the spec mandate?
-
-> *answer:*
-
-**5. Should we add a version byte after the magic?**
-Currently `ZNAVSRFG` is followed immediately by protobuf. Adding a version byte (`ZNAVSRFG\x01`) would allow future framing changes without requiring protobuf parsing to detect the version. `MFFileOuter.Version` serves this purpose but requires successful deserialization to read. Worth the extra byte?
-
-> *answer:*
-
-**6. Should we add a length-prefix after the magic?**
-Protobuf is not self-delimiting. If we ever want to concatenate manifests or append data after the protobuf, the current framing is insufficient. Add a varint or fixed-width length-prefix?
-
-> *answer:*
-
-### Signature Design
-
-**7. What does the outer SHA-256 hash cover — compressed or uncompressed data?**
-The review notes it currently hashes compressed data (good for verifying before decompression), but this should be explicitly documented. Which is the intended behavior?
-
-> *answer:*
-
-**8. Should `signatureString()` sign raw bytes instead of a hex-encoded string?**
-Currently the canonical string is `MAGIC-UUID-MULTIHASH` with hex encoding, which adds a transformation layer. Signing the raw `sha256` bytes (or compressed `innerMessage` directly) would be simpler. Keep the string format or switch to raw bytes?
-
-> *answer:*
-
-**9. Should we support detached signature files (`.mf.sig`)?**
-Embedded signatures are better for single-file distribution. Detached `.mf.sig` files follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are simpler for HTTP serving. Support both modes?
-
-> *answer:*
-
-**10. GPG vs pure-Go crypto for signatures?**
-Shelling out to `gpg` is fragile (may not be installed, version-dependent output). `github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could go Ed25519/signify (simpler, no key management). Which direction?
-
-> *answer:*
-
-### Implementation Design
-
-**11. Should manifests be deterministic by default?**
-This means: sort file entries by path, omit `createdAt` timestamp (or make it opt-in), no `atime`. Should determinism be the default, with a `--include-timestamps` flag to opt in?
-
-> *answer:*
-
-**12. Should we consolidate or keep both scanner/checker implementations?**
-There are two parallel implementations: `mfer/scanner.go` + `mfer/checker.go` (typed with `FileSize`, `RelFilePath`) and `internal/scanner/` + `internal/checker/` (raw `int64`, `string`). The `mfer/` versions are superior. Delete the `internal/` versions?
-
-> *answer:*
-
-**13. Should the `manifest` type be exported?**
-Currently unexported with exported constructors (`New`, `NewFromPaths`, etc.). Consumers can't declare `var m *mfer.manifest`. Export the type, or define an interface?
-
-> *answer:*
-
-**14. What should the Go module path be for 1.0?**
-Currently mixed between `sneak.berlin/go/mfer` and `git.eeqj.de/sneak/mfer`. Which is canonical?
-
-> *answer:*
-
---
-
-## Implementation Plan
-
-### Phase 1: Foundation (format correctness)
-
- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate on `mfer/` package versions; update CLI code
- [ ] Add deterministic file ordering — sort entries by path (lexicographic, byte-order) in `Builder.Build()`; add test asserting byte-identical output from two runs
- [ ] Add decompression size limit — `io.LimitReader` in `deserializeInner()` with `m.pbOuter.Size` as bound
- [ ] Fix `errors.Is` dead code in checker — replace with `os.IsNotExist(err)` or `errors.Is(err, fs.ErrNotExist)`
- [ ] Fix `AddFile` to verify size — check `totalRead == size` after reading, return error on mismatch
- [ ] Specify path invariants — add proto comments (UTF-8, forward-slash, relative, no `..`, no leading `/`); validate in `Builder.AddFile` and `Builder.AddFileWithHash`
-
-### Phase 2: CLI polish
-
- [ ] Fix flag naming — all CLI flags use kebab-case as primary (`--include-dotfiles`, `--follow-symlinks`)
- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or `url.JoinPath()` instead of string concatenation
- [ ] Add progress rate-limiting to Checker — throttle to once per second, matching Scanner
- [ ] Add `--deterministic` flag (or make it default) — omit `createdAt`, sort files
-
-### Phase 3: Robustness
-
- [ ] Replace GPG subprocess with pure-Go crypto — `github.com/ProtonMail/go-crypto` or Ed25519/signify
- [ ] Add timeout to any remaining subprocess calls
- [ ] Add fuzzing tests for `NewManifestFromReader`
- [ ] Add retry logic to fetch — exponential backoff for transient HTTP errors
-
-### Phase 4: Format finalization
-
- [ ] Remove or deprecate `atime` from proto (pending design question answer)
- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future file permissions
- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version 1
- [ ] Write format specification document — separate from README: magic, outer structure, compression, inner structure, path invariants, signature scheme, canonical serialization
-
-### Phase 5: Release prep
-
- [ ] Finalize Go module path
- [ ] Audit all error messages for consistency and helpfulness
- [ ] Add `--version` output matching SemVer
- [ ] Tag v1.0.0
--- a/bin/gitrev.sh
+++ b/bin/gitrev.sh
@@ -1,12 +0,0 @@
-#!/bin/bash
-#
-if [[ ! -z "$DRONE_COMMIT_SHA" ]]; then
-    echo "${DRONE_COMMIT_SHA:0:7}"
-    exit 0
-fi
-
-if [[ ! -z "$GITREV" ]]; then
-    echo $GITREV
-else
-    git describe --always --dirty=-dirty
-fi
--- a/cmd/mfer/main.go
+++ b/cmd/mfer/main.go
@@ -9,9 +9,8 @@ import (
 var (
 	Appname string = "mfer"
 	Version string
-	Gitrev  string
 )

 func main() {
-	os.Exit(cli.Run(Appname, Version, Gitrev))
+	os.Exit(cli.Run(Appname, Version))
 }
--- a/cmd/mfer/main_test.go
+++ b/cmd/mfer/main_test.go
@@ -1,11 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBuild(t *testing.T) {
-	assert.True(t, true)
-}
--- a/go.mod
+++ b/go.mod
@@ -1,33 +1,12 @@
 module git.eeqj.de/sneak/mfer

-go 1.22
+go 1.16

 require (
-	github.com/apex/log v1.9.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/pterm/pterm v0.12.35
 	github.com/spf13/afero v1.8.0
-	github.com/stretchr/testify v1.8.1
-	github.com/urfave/cli/v2 v2.23.6
-	google.golang.org/protobuf v1.28.1
-)
-
-require (
-	github.com/atomicgo/cursor v0.0.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/fatih/color v1.7.0 // indirect
-	github.com/gookit/color v1.4.2 // indirect
-	github.com/mattn/go-colorable v0.1.2 // indirect
-	github.com/mattn/go-isatty v0.0.8 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
-	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.4 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	github.com/urfave/cli/v2 v2.3.0
+	github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 // indirect
+	google.golang.org/protobuf v1.27.1
 )
--- a/go.sum
+++ b/go.sum
@@ -44,15 +44,8 @@ github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBE
 github.com/MarvinJWendt/testza v0.2.10/go.mod h1:pd+VWsoGUiFtq+hRKSU1Bktnn+DMCSrDrXDpX2bG66k=
 github.com/MarvinJWendt/testza v0.2.12 h1:/PRp/BF+27t2ZxynTiqj0nyND5PbOtfJS0SuTuxmgeg=
 github.com/MarvinJWendt/testza v0.2.12/go.mod h1:JOIegYyV7rX+7VZ9r77L/eH6CfJHHzXjB69adAhzZkI=
-github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
-github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
-github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
-github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
-github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
 github.com/atomicgo/cursor v0.0.1 h1:xdogsqa6YYlLfM+GyClC/Lchf7aiMerFiZQn7soTOoU=
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
-github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -61,8 +54,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -72,13 +65,9 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -132,7 +121,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@@ -141,37 +129,19 @@ github.com/gookit/color v1.4.2 h1:tXy44JFSFkKnELV6WaMo/lLfu/meqITX3iAV52do7lk=
 github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -186,41 +156,25 @@ github.com/pterm/pterm v0.12.35 h1:A/vHwDM+WByn0sTPlpL2L6kOTy12xqZuwNFMF/NlA+U=
 github.com/pterm/pterm v0.12.35/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
-github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
-github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60=
 github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
-github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
-github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
-github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
-github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
-github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
-github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
-github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
-github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
+github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
+github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 h1:RDJ3ggSqBL4Mu/ANRKmxuLrwnupJsvHRDXe4/KI+HWE=
+github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20/go.mod h1:iljxuLc3m07jsXOWojqehoTp/Fh0XP7irbhV+6sYNGc=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -232,7 +186,6 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -274,7 +227,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -324,9 +276,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -518,22 +468,16 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
--- a/internal/bork/error.go
+++ b/internal/bork/error.go
@@ -1,15 +0,0 @@
-package bork
-
-import (
-	"errors"
-	"fmt"
-)
-
-var (
-	ErrMissingMagic  = errors.New("missing magic bytes in file")
-	ErrFileTruncated = errors.New("file/stream is truncated abnormally")
-)
-
-func Newf(format string, args ...interface{}) error {
-	return fmt.Errorf(format, args...)
-}
--- a/internal/bork/error_test.go
+++ b/internal/bork/error_test.go
@@ -1,11 +0,0 @@
-package bork
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBuild(t *testing.T) {
-	assert.NotNil(t, ErrMissingMagic)
-}
--- a/internal/cli/check.go
+++ b/internal/cli/check.go
@@ -1,13 +0,0 @@
-package cli
-
-import (
-	"errors"
-
-	"github.com/apex/log"
-	"github.com/urfave/cli/v2"
-)
-
-func (mfa *CLIApp) checkManifestOperation(c *cli.Context) error {
-	log.WithError(errors.New("unimplemented"))
-	return nil
-}
--- a/internal/cli/entry.go
+++ b/internal/cli/entry.go
@@ -13,11 +13,10 @@ func init() {
 	}
 }

-func Run(Appname, Version, Gitrev string) int {
+func Run(Appname, Version string) int {
 	m := &CLIApp{}
 	m.appname = Appname
 	m.version = Version
-	m.gitrev = Gitrev
 	m.exitCode = 0

 	m.run()
--- a/internal/cli/entry_test.go
+++ b/internal/cli/entry_test.go
@@ -1,12 +0,0 @@
-package cli
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBuild(t *testing.T) {
-	m := &CLIApp{}
-	assert.NotNil(t, m)
-}
--- a/internal/cli/fetch.go
+++ b/internal/cli/fetch.go
@@ -1,12 +0,0 @@
-package cli
-
-import (
-	"github.com/apex/log"
-	"github.com/urfave/cli/v2"
-)
-
-func (mfa *CLIApp) fetchManifestOperation(c *cli.Context) error {
-	log.Debugf("fetchManifestOperation()")
-	panic("not implemented")
-	return nil //nolint
-}
--- a/internal/cli/gen.go
+++ b/internal/cli/gen.go
@@ -1,54 +0,0 @@
-package cli
-
-import (
-	"bytes"
-	"path/filepath"
-
-	"git.eeqj.de/sneak/mfer/internal/log"
-	"git.eeqj.de/sneak/mfer/mfer"
-	"github.com/urfave/cli/v2"
-)
-
-func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
-	log.Debug("generateManifestOperation()")
-	myArgs := ctx.Args()
-	log.Dump(myArgs)
-
-	opts := &mfer.ManifestScanOptions{
-		IgnoreDotfiles: ctx.Bool("IgnoreDotfiles"),
-		FollowSymLinks: ctx.Bool("FollowSymLinks"),
-	}
-	paths := make([]string, ctx.Args().Len()-1)
-	for i := 0; i < ctx.Args().Len(); i++ {
-		ap, err := filepath.Abs(ctx.Args().Get(i))
-		if err != nil {
-			return err
-		}
-		log.Dump(ap)
-		paths = append(paths, ap)
-	}
-	mf, err := mfer.NewFromPaths(opts, paths...)
-	if err != nil {
-		panic(err)
-	}
-	mf.WithContext(ctx.Context)
-
-	log.Dump(mf)
-
-	err = mf.Scan()
-	if err != nil {
-		return err
-	}
-
-	buf := new(bytes.Buffer)
-
-	err = mf.WriteTo(buf)
-	if err != nil {
-		return err
-	}
-
-	dat := buf.Bytes()
-
-	log.Dump(dat)
-	return nil
-}
--- a/internal/cli/manifest.go
+++ b/internal/cli/manifest.go
@@ -0,0 +1,51 @@
+package cli
+
+import (
+	"git.eeqj.de/sneak/mfer/mfer"
+	"github.com/spf13/afero"
+)
+
+type Job struct {
+	innerpb   *mfer.MFFileInner
+	outerpb   *mfer.MFFile
+	fileCount int64
+	totalSize int64
+	afs       afero.Fs
+}
+
+func (m *Job) scanForFiles() error {
+
+	m.innerpb = &mfer.MFFileInner{}
+	m.innerpb.Version = mfer.MFFileInner_ONE
+	return nil
+}
+
+/*
+	walkErr := filepath.Walk(m.sourcePath, func(itemPath string, info os.FileInfo, err error) error {
+
+		// we do not include the manifest file in the manifest
+		if itemPath == "index.mf" {
+			return nil
+		}
+
+		fpi := mfer.MFFilePath{}
+		fpi.Path = itemPath
+		fpi.Size = info.Size()
+		m.innerpb.Files = append(m.innerpb.Files, &fpi)
+		m.fileCount++
+		m.totalSize += fpi.Size
+		return nil
+	})
+
+	if walkErr != nil {
+		log.Fatal(walkErr)
+		return walkErr
+	}
+
+	fmt.Printf("%#v\n", m.innerpb)
+	fmt.Printf("filecount = %#v\n", m.fileCount)
+	fmt.Printf("totalsize = %#v\n", m.totalSize)
+	return nil
+}
+
+*/
--- a/internal/cli/mfer.go
+++ b/internal/cli/mfer.go
@@ -1,104 +1,73 @@
 package cli

 import (
+	"errors"
 	"fmt"
 	"os"
 	"time"

-	"git.eeqj.de/sneak/mfer/internal/log"
+	log "github.com/visionmedia/go-cli-log"
+
+	"git.eeqj.de/sneak/mfer/mfer"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/pterm/pterm"
 	"github.com/urfave/cli/v2"
 )

 type CLIApp struct {
 	appname     string
 	version     string
-	gitrev      string
+	buildarch   string
 	startupTime time.Time
 	exitCode    int
+	errorString string
 	app         *cli.App
 }

-const banner = `              ___           ___         ___           ___
-	     /__/\         /  /\       /  /\         /  /\
-	    |  |::\       /  /:/_     /  /:/_       /  /::\
-	    |  |:|:\     /  /:/ /\   /  /:/ /\     /  /:/\:\
-	  __|__|:|\:\   /  /:/ /:/  /  /:/ /:/_   /  /:/~/:/
-	 /__/::::| \:\ /__/:/ /:/  /__/:/ /:/ /\ /__/:/ /:/___
-	 \  \:\~~\__\/ \  \:\/:/   \  \:\/:/ /:/ \  \:\/:::::/
-	  \  \:\        \  \::/     \  \::/ /:/   \  \::/~~~~
-	   \  \:\        \  \:\      \  \:\/:/     \  \:\
-	    \  \:\        \  \:\      \  \::/       \  \:\
-	     \__\/         \__\/       \__\/         \__\/`
-
 func (mfa *CLIApp) printBanner() {
-	fmt.Println(banner)
+	s, _ := pterm.DefaultBigText.WithLetters(pterm.NewLettersFromString(mfa.appname)).Srender()
+	pterm.DefaultCenter.Println(s) // Print BigLetters with the default CenterPrinter
 }

-func (mfa *CLIApp) VersionString() string {
-	return fmt.Sprintf("%s (%s)", mfa.version, mfa.gitrev)
-}
-
-func (mfa *CLIApp) setVerbosity(v int) {
-	_, present := os.LookupEnv("MFER_DEBUG")
-	if present {
-		log.EnableDebugLogging()
-	} else {
-		log.SetLevelFromVerbosity(v)
-	}
+func (mfa *CLIApp) disableStyling() {
+	pterm.DisableColor()
+	pterm.DisableStyling()
+	pterm.Debug.Prefix.Text = ""
+	pterm.Info.Prefix.Text = ""
+	pterm.Success.Prefix.Text = ""
+	pterm.Warning.Prefix.Text = ""
+	pterm.Error.Prefix.Text = ""
+	pterm.Fatal.Prefix.Text = ""
 }

 func (mfa *CLIApp) run() {
-	mfa.startupTime = time.Now()

 	if NO_COLOR {
 		// shoutout to rob pike who thinks it's juvenile
-		log.DisableStyling()
+		mfa.disableStyling()
 	}

-	log.Init()
-
-	var verbosity int
+	mfa.printBanner()

 	mfa.app = &cli.App{
 		Name:                 mfa.appname,
 		Usage:                "Manifest generator",
-		Version:              mfa.VersionString(),
+		Version:              mfa.version,
 		EnableBashCompletion: true,
-		Flags: []cli.Flag{
-			&cli.BoolFlag{
-				Name:    "verbose",
-				Usage:   "Verbosity level",
-				Aliases: []string{"v"},
-				Count:   &verbosity,
-			},
-			&cli.BoolFlag{
-				Name:    "quiet",
-				Usage:   "don't produce output except on error",
-				Aliases: []string{"q"},
-			},
-		},
 		Commands: []*cli.Command{
 			{
 				Name:    "generate",
 				Aliases: []string{"gen"},
 				Usage:   "Generate manifest file",
 				Action: func(c *cli.Context) error {
-					if !c.Bool("quiet") {
-						mfa.printBanner()
-					}
-					mfa.setVerbosity(verbosity)
 					return mfa.generateManifestOperation(c)
 				},
 				Flags: []cli.Flag{
-					&cli.BoolFlag{
-						Name:    "FollowSymLinks",
-						Aliases: []string{"follow-symlinks"},
-						Usage:   "Resolve encountered symlinks",
-					},
-					&cli.BoolFlag{
-						Name:    "IgnoreDotfiles",
-						Aliases: []string{"ignore-dotfiles"},
-						Usage:   "Ignore any dot (hidden) files encountered",
+					&cli.StringFlag{
+						Name:    "input",
+						Value:   ".",
+						Aliases: []string{"i"},
+						Usage:   "Specify input directory.",
 					},
 					&cli.StringFlag{
 						Name:    "output",
@@ -112,39 +81,51 @@ func (mfa *CLIApp) run() {
 				Name:  "check",
 				Usage: "Validate files using manifest file",
 				Action: func(c *cli.Context) error {
-					if !c.Bool("quiet") {
-						mfa.printBanner()
-					}
-					mfa.setVerbosity(verbosity)
-					return mfa.checkManifestOperation(c)
-				},
-			},
-			{
-				Name:  "version",
-				Usage: "Show version",
-				Action: func(c *cli.Context) error {
-					fmt.Printf("%s\n", mfa.VersionString())
-					return nil
-				},
-			},
-			{
-				Name:  "fetch",
-				Usage: "fetch manifest and referenced files",
-				Action: func(c *cli.Context) error {
-					if !c.Bool("quiet") {
-						mfa.printBanner()
-					}
-					mfa.setVerbosity(verbosity)
-					return mfa.fetchManifestOperation(c)
+					return mfa.validateManifestOperation(c)
 				},
 			},
 		},
 	}

-	mfa.app.HideVersion = true
 	err := mfa.app.Run(os.Args)
+
 	if err != nil {
 		mfa.exitCode = 1
-		log.WithError(err).Debugf("exiting")
+		log.Error(err)
 	}
 }
+
+func (mfa *CLIApp) validateManifestOperation(c *cli.Context) error {
+	log.Error(errors.New("unimplemented"))
+	return nil
+}
+
+func (mfa *CLIApp) generateManifestOperation(c *cli.Context) error {
+	fmt.Println("generateManifestOperation()")
+	fmt.Printf("called with arg: %s\n", c.String("input"))
+
+	opts := &mfer.ManifestScanOptions{
+		IgnoreDotfiles: true,
+		FollowSymLinks: false,
+	}
+	mf, err := mfer.NewFromPath(c.String("input"), opts)
+
+	if err != nil {
+		panic(err)
+	}
+
+	spew.Dump(mf)
+
+	/*
+
+		mgj, err := NewMFGenerationJobFromFilesystem(c.String("input"))
+		if err != nil {
+			log.Fatal(err)
+			return err
+		}
+		mgj.scanForFiles()
+		//mgj.outputFile = c.String("output")
+
+	*/
+	return nil
+}
--- a/internal/log/log.go
+++ b/internal/log/log.go
@@ -1,89 +0,0 @@
-package log
-
-import (
-	"fmt"
-	"runtime"
-
-	"github.com/apex/log"
-	acli "github.com/apex/log/handlers/cli"
-	"github.com/davecgh/go-spew/spew"
-	"github.com/pterm/pterm"
-)
-
-type Level = log.Level
-
-func DisableStyling() {
-	pterm.DisableColor()
-	pterm.DisableStyling()
-	pterm.Debug.Prefix.Text = ""
-	pterm.Info.Prefix.Text = ""
-	pterm.Success.Prefix.Text = ""
-	pterm.Warning.Prefix.Text = ""
-	pterm.Error.Prefix.Text = ""
-	pterm.Fatal.Prefix.Text = ""
-}
-
-func Init() {
-	log.SetHandler(acli.Default)
-	log.SetLevel(log.InfoLevel)
-}
-
-func Debugf(format string, args ...interface{}) {
-	DebugReal(fmt.Sprintf(format, args...), 2)
-}
-
-func Debug(arg string) {
-	DebugReal(arg, 2)
-}
-
-func DebugReal(arg string, cs int) {
-	_, callerFile, callerLine, ok := runtime.Caller(cs)
-	if !ok {
-		return
-	}
-	tag := fmt.Sprintf("%s:%d: ", callerFile, callerLine)
-	log.Debug(tag + arg)
-}
-
-func Dump(args ...interface{}) {
-	DebugReal(spew.Sdump(args...), 2)
-}
-
-func EnableDebugLogging() {
-	SetLevel(log.DebugLevel)
-}
-
-func VerbosityStepsToLogLevel(l int) log.Level {
-	switch l {
-	case 1:
-		return log.WarnLevel
-	case 2:
-		return log.InfoLevel
-	case 3:
-		return log.DebugLevel
-	}
-	return log.ErrorLevel
-}
-
-func SetLevelFromVerbosity(l int) {
-	SetLevel(VerbosityStepsToLogLevel(l))
-}
-
-func SetLevel(arg log.Level) {
-	log.SetLevel(arg)
-}
-
-func GetLogger() *log.Logger {
-	if logger, ok := log.Log.(*log.Logger); ok {
-		return logger
-	}
-	panic("unable to get logger")
-}
-
-func GetLevel() log.Level {
-	return GetLogger().Level
-}
-
-func WithError(e error) *log.Entry {
-	return GetLogger().WithError(e)
-}
--- a/internal/log/log_test.go
+++ b/internal/log/log_test.go
@@ -1,12 +0,0 @@
-package log
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBuild(t *testing.T) {
-	Init()
-	assert.True(t, true)
-}
--- a/mfer/deserialize.go
+++ b/mfer/deserialize.go
@@ -1,89 +0,0 @@
-package mfer
-
-import (
-	"bytes"
-	"compress/gzip"
-	"errors"
-	"io"
-
-	"git.eeqj.de/sneak/mfer/internal/bork"
-	"git.eeqj.de/sneak/mfer/internal/log"
-	"google.golang.org/protobuf/proto"
-)
-
-func (m *manifest) validateProtoOuter() error {
-	if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
-		return errors.New("unknown version")
-	}
-	if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
-		return errors.New("unknown compression type")
-	}
-
-	bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
-
-	gzr, err := gzip.NewReader(bb)
-	if err != nil {
-		return err
-	}
-
-	dat, err := io.ReadAll(gzr)
-	defer gzr.Close()
-
-	if err != nil {
-		return err
-	}
-
-	isize := len(dat)
-	if int64(isize) != m.pbOuter.Size {
-		log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
-		return bork.ErrFileTruncated
-	}
-	log.Debugf("inner data size is %d", isize)
-	log.Dump(dat)
-	log.Dump(m.pbOuter.Sha256)
-	return nil
-}
-
-func validateMagic(dat []byte) bool {
-	ml := len([]byte(MAGIC))
-	if len(dat) < ml {
-		return false
-	}
-	got := dat[0:ml]
-	expected := []byte(MAGIC)
-	return bytes.Equal(got, expected)
-}
-
-func NewFromProto(input io.Reader) (*manifest, error) {
-	m := New()
-	dat, err := io.ReadAll(input)
-	if err != nil {
-		return nil, err
-	}
-	if !validateMagic(dat) {
-		return nil, errors.New("invalid file format")
-	}
-
-	// remove magic bytes prefix:
-	ml := len([]byte(MAGIC))
-	bb := bytes.NewBuffer(dat[ml:])
-	dat = bb.Bytes()
-
-	log.Dump(dat)
-
-	// deserialize:
-	m.pbOuter = new(MFFileOuter)
-	err = proto.Unmarshal(dat, m.pbOuter)
-
-	if err != nil {
-		return nil, err
-	}
-
-	ve := m.validateProtoOuter()
-	if ve != nil {
-		return nil, ve
-	}
-
-	// FIXME TODO deserialize inner
-	return m, nil
-}
--- a/mfer/example_test.go
+++ b/mfer/example_test.go
@@ -1,42 +0,0 @@
-package mfer
-
-import (
-	"bytes"
-	"testing"
-
-	"git.eeqj.de/sneak/mfer/internal/log"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestAPIExample(t *testing.T) {
-	// read from filesystem
-	m, err := NewFromFS(&ManifestScanOptions{
-		IgnoreDotfiles: true,
-	}, big)
-	assert.Nil(t, err)
-	assert.NotNil(t, m)
-
-	// scan for files
-	m.Scan()
-
-	// serialize
-	var buf bytes.Buffer
-	m.WriteTo(&buf)
-
-	// show serialized
-	log.Dump(buf.Bytes())
-
-	// do it again
-	var buf2 bytes.Buffer
-	m.WriteTo(&buf2)
-
-	// should be same!
-	assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
-
-	// deserialize
-	m2, err := NewFromProto(&buf)
-	assert.Nil(t, err)
-	assert.NotNil(t, m2)
-
-	log.Dump(m2)
-}
--- a/mfer/gen.go
+++ b/mfer/gen.go
@@ -0,0 +1,3 @@
+package mfer
+
+//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
--- a/mfer/manifest.go
+++ b/mfer/manifest.go
@@ -1,42 +1,31 @@
 package mfer

 import (
-	"bytes"
-	"context"
-	"errors"
 	"fmt"
+	"io"
 	"io/fs"
-	"path"
 	"path/filepath"
 	"strings"

-	"git.eeqj.de/sneak/mfer/internal/log"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/spf13/afero"
 )

-type manifestFile struct {
-	path string
-	info fs.FileInfo
+type ManifestFile struct {
+	Path     string
+	FileInfo fs.FileInfo
 }

-func (m *manifestFile) String() string {
-	return fmt.Sprintf("<File \"%s\">", m.path)
+func (m *ManifestFile) String() string {
+	return fmt.Sprintf("<File \"%s\">", m.Path)
 }

-type manifest struct {
-	sourceFS      []afero.Fs
-	files         []*manifestFile
-	scanOptions   *ManifestScanOptions
-	totalFileSize int64
-	pbInner       *MFFile
-	pbOuter       *MFFileOuter
-	output        *bytes.Buffer
-	ctx           context.Context
-	errors        []*error
-}
-
-func (m *manifest) String() string {
-	return fmt.Sprintf("<Manifest count=%d totalSize=%d>", len(m.files), m.totalFileSize)
+type Manifest struct {
+	SourceFS      afero.Fs
+	SourceFSRoot  string
+	Files         []*ManifestFile
+	ScanOptions   *ManifestScanOptions
+	TotalFileSize int64
 }

 type ManifestScanOptions struct {
@@ -44,130 +33,65 @@ type ManifestScanOptions struct {
 	FollowSymLinks bool
 }

-func (m *manifest) HasError() bool {
-	return len(m.errors) > 0
-}
-
-func (m *manifest) AddError(e error) *manifest {
-	m.errors = append(m.errors, &e)
-	return m
-}
-
-func (m *manifest) WithContext(c context.Context) *manifest {
-	m.ctx = c
-	return m
-}
-
-func (m *manifest) addInputPath(inputPath string) error {
+func NewFromPath(inputPath string, options *ManifestScanOptions) (*Manifest, error) {
 	abs, err := filepath.Abs(inputPath)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	// FIXME check to make sure inputPath/abs exists maybe
-	afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs))
-	return m.addInputFS(afs)
-}
-
-func (m *manifest) addInputFS(f afero.Fs) error {
-	if m.sourceFS == nil {
-		m.sourceFS = make([]afero.Fs, 0)
-	}
-	m.sourceFS = append(m.sourceFS, f)
-	// FIXME do some sort of check on f here?
-	return nil
-}
-
-func New() *manifest {
-	m := &manifest{}
-	return m
-}
-
-func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) {
-	log.Dump(inputPaths)
-	m := New()
-	m.scanOptions = options
-	for _, p := range inputPaths {
-		err := m.addInputPath(p)
+	afs := afero.NewBasePathFs(afero.NewOsFs(), abs)
+	spew.Dump(afs)
+	m, err := NewFromFS(afs, options)
 	if err != nil {
 		return nil, err
 	}
-	}
+	m.SourceFSRoot = abs
+	spew.Dump(m)
 	return m, nil
 }

-func NewFromFS(options *ManifestScanOptions, fs afero.Fs) (*manifest, error) {
-	m := New()
-	m.scanOptions = options
-	err := m.addInputFS(fs)
-	if err != nil {
-		return nil, err
+func NewFromFS(fs afero.Fs, options *ManifestScanOptions) (*Manifest, error) {
+	m := &Manifest{
+		SourceFS:    fs,
+		ScanOptions: options,
 	}
+	m.Scan()
 	return m, nil
 }

-func (m *manifest) GetFileCount() int64 {
-	return int64(len(m.files))
-}
+func (m *Manifest) Scan() {
+	afero.Walk(m.SourceFS, "./", func(path string, info fs.FileInfo, err error) error {

-func (m *manifest) GetTotalFileSize() int64 {
-	return m.totalFileSize
-}
-
-func pathIsHidden(p string) bool {
-	tp := path.Clean(p)
-	if strings.HasPrefix(tp, ".") {
-		return true
-	}
-	for {
-		d, f := path.Split(tp)
-		if strings.HasPrefix(f, ".") {
-			return true
-		}
-		if d == "" {
-			return false
-		}
-		tp = d[0 : len(d)-1] // trim trailing slash from dir
-	}
-}
-
-func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
-	if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) {
+		if m.ScanOptions.IgnoreDotfiles && strings.HasPrefix(path, ".") {
+			// FIXME make this check all path components BUG
 			return nil
 		}
-	if fi != nil && fi.IsDir() {
+
+		if info != nil && info.IsDir() {
 			// manifests contain only files, directories are implied.
 			return nil
 		}
-	// FIXME test if 'fi' is already result of stat
-	fileinfo, staterr := m.sourceFS[sfsIndex].Stat(p)
+
+		fmt.Printf("path = %s\n", path)
+		spew.Dump(path)
+		spew.Dump(info)
+
+		fileinfo, staterr := m.SourceFS.Stat(path)
 		if staterr != nil {
-		return staterr
-	}
-	cleanPath := p
-	if cleanPath[0:1] == "/" {
-		cleanPath = cleanPath[1:]
-	}
-	nf := &manifestFile{
-		path: cleanPath,
-		info: fileinfo,
-	}
-	m.files = append(m.files, nf)
-	m.totalFileSize = m.totalFileSize + fi.Size()
-	return nil
+			panic(staterr)
 		}

-func (m *manifest) Scan() error {
-	// FIXME scan and whatever function does the hashing should take ctx
-	for idx, sfs := range m.sourceFS {
-		if sfs == nil {
-			return errors.New("invalid source fs")
+		nf := &ManifestFile{
+			Path:     path,
+			FileInfo: fileinfo,
 		}
-		e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error {
-			return m.addFile(p, info, idx)
+		m.Files = append(m.Files, nf)
+		m.TotalFileSize = m.TotalFileSize + info.Size()
+		fmt.Printf("total file count now %i\n", len(m.Files))
+		fmt.Printf("total file size now %i\n", m.TotalFileSize)
+		return nil
 	})
-		if e != nil {
-			return e
-		}
 }
+
+func (m *Manifest) Write(output io.Writer) error {
 	return nil
 }
--- a/mfer/mf.proto
+++ b/mfer/mf.proto
@@ -1,34 +1,26 @@
 syntax = "proto3";

-option go_package = "git.eeqj.de/sneak/mfer/mfer";
+option go_package = "git.eeqj.de/sneak/mfer";

 message Timestamp {
  int64 seconds = 1;
  int32 nanos = 2;
 }

-message MFFileOuter {
+message MFFile {
    enum Version {
-        VERSION_NONE = 0;
-        VERSION_ONE = 1; // only one for now
+        NONE = 0;
+        ONE = 1; // only one for now
    }

    // required mffile root attributes 1xx
-    Version version = 101;
-
-    enum CompressionType {
-        COMPRESSION_NONE = 0;
-        COMPRESSION_GZIP = 1;
-    }
-
-    CompressionType compressionType = 102;
-
+    Version version = 1;
+    bytes innerMessage = 2;
    // these are used solely to detect corruption/truncation
    // and not for cryptographic integrity.
-    int64 size = 103;
-    bytes sha256 = 104;
+    int64 size = 3;
+    bytes sha256 = 4;

-    bytes innerMessage = 199;
    // 2xx for optional manifest root attributes
    // think we might use gosignify instead of gpg:
    // github.com/frankbraun/gosignify
@@ -62,15 +54,15 @@ message MFFileChecksum {
    bytes multiHash = 1;
 }

-message MFFile {
+message MFFileInner {
    enum Version {
-        VERSION_NONE = 0;
-        VERSION_ONE = 1; // only one for now
+        NONE = 0;
+        ONE = 1; // only one for now
    }
-    Version version = 100;
+    Version version = 1;

    // required manifest attributes:
-    repeated MFFilePath files = 101;
+    repeated MFFilePath files = 2;

    // optional manifest attributes 2xx:
    optional Timestamp createdAt = 201;
--- a/mfer/mfer_test.go
+++ b/mfer/mfer_test.go
@@ -1,74 +1,13 @@
 package mfer

-import (
-	"bytes"
-	"fmt"
-	"testing"
-
-	"git.eeqj.de/sneak/mfer/internal/log"
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
-)
+import "testing"

 // Add those variables as well
 var (
 	existingFolder    = "./testdata/a-folder-that-exists"
+	nonExistingFolder = "./testdata/a-folder-that-does-not-exists"
 )

-var (
-	af  *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
-	big *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
-)
+func TestManifestGeneration(t *testing.T) {

-func init() {
-	log.EnableDebugLogging()
-
-	// create test files and directories
-	af.MkdirAll("/a/b/c", 0o755)
-	af.MkdirAll("/.hidden", 0o755)
-	af.WriteFile("/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
-	af.WriteFile("/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
-	af.WriteFile("/.hidden/hello.txt", []byte("hello world\n"), 0o755)
-	af.WriteFile("/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
-
-	big.MkdirAll("/home/user/Library", 0o755)
-	for i := range [25]int{} {
-		big.WriteFile(fmt.Sprintf("/home/user/Library/hello%d.txt", i), []byte("hello world\n"), 0o755)
-	}
-}
-
-func TestPathHiddenFunc(t *testing.T) {
-	assert.False(t, pathIsHidden("/a/b/c/hello.txt"))
-	assert.True(t, pathIsHidden("/a/b/c/.hello.txt"))
-	assert.True(t, pathIsHidden("/a/.b/c/hello.txt"))
-	assert.True(t, pathIsHidden("/.a/b/c/hello.txt"))
-	assert.False(t, pathIsHidden("./a/b/c/hello.txt"))
-}
-
-func TestManifestGenerationOne(t *testing.T) {
-	m, err := NewFromFS(&ManifestScanOptions{
-		IgnoreDotfiles: true,
-	}, af)
-	assert.Nil(t, err)
-	assert.NotNil(t, m)
-	m.Scan()
-	assert.Equal(t, int64(2), m.GetFileCount())
-	assert.Equal(t, int64(30), m.GetTotalFileSize())
-}
-
-func TestManifestGenerationTwo(t *testing.T) {
-	m, err := NewFromFS(&ManifestScanOptions{
-		IgnoreDotfiles: false,
-	}, af)
-	assert.Nil(t, err)
-	assert.NotNil(t, m)
-	m.Scan()
-	assert.Equal(t, int64(4), m.GetFileCount())
-	assert.Equal(t, int64(54), m.GetTotalFileSize())
-	err = m.generate()
-	assert.Nil(t, err)
-	var buf bytes.Buffer
-	err = m.WriteTo(&buf)
-	assert.Nil(t, err)
-	log.Dump(buf.Bytes())
 }
--- a/mfer/output.go
+++ b/mfer/output.go
@@ -1,33 +0,0 @@
-package mfer
-
-import (
-	"io"
-	"os"
-)
-
-func (m *manifest) WriteToFile(path string) error {
-	// FIXME refuse to overwrite without -f if file exists
-
-	f, err := os.Create(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	return m.WriteTo(f)
-}
-
-func (m *manifest) WriteTo(output io.Writer) error {
-	if m.pbOuter == nil {
-		err := m.generate()
-		if err != nil {
-			return err
-		}
-	}
-
-	_, err := output.Write(m.output.Bytes())
-	if err != nil {
-		return err
-	}
-	return nil
-}
--- a/mfer/serialize.go
+++ b/mfer/serialize.go
@@ -1,100 +0,0 @@
-package mfer
-
-import (
-	"bytes"
-	"compress/gzip"
-	"crypto/sha256"
-	"errors"
-	"time"
-
-	"google.golang.org/protobuf/proto"
-)
-
-//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
-
-// rot13("MANIFEST")
-const MAGIC string = "ZNAVSRFG"
-
-func newTimestampFromTime(t time.Time) *Timestamp {
-	out := &Timestamp{
-		Seconds: t.Unix(),
-		Nanos:   int32(t.UnixNano() - (t.Unix() * 1000000000)),
-	}
-	return out
-}
-
-func (m *manifest) generate() error {
-	if m.pbInner == nil {
-		e := m.generateInner()
-		if e != nil {
-			return e
-		}
-	}
-	if m.pbOuter == nil {
-		e := m.generateOuter()
-		if e != nil {
-			return e
-		}
-	}
-	dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
-	if err != nil {
-		return err
-	}
-	m.output = bytes.NewBuffer([]byte(MAGIC))
-	_, err = m.output.Write(dat)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (m *manifest) generateOuter() error {
-	if m.pbInner == nil {
-		return errors.New("internal error")
-	}
-	innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
-	if err != nil {
-		return err
-	}
-
-	h := sha256.New()
-	h.Write(innerData)
-
-	idc := new(bytes.Buffer)
-	gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
-	if err != nil {
-		return err
-	}
-	_, err = gzw.Write(innerData)
-	if err != nil {
-		return err
-	}
-
-	gzw.Close()
-
-	o := &MFFileOuter{
-		InnerMessage:    idc.Bytes(),
-		Size:            int64(len(innerData)),
-		Sha256:          h.Sum(nil),
-		Version:         MFFileOuter_VERSION_ONE,
-		CompressionType: MFFileOuter_COMPRESSION_GZIP,
-	}
-	m.pbOuter = o
-	return nil
-}
-
-func (m *manifest) generateInner() error {
-	m.pbInner = &MFFile{
-		Version:   MFFile_VERSION_ONE,
-		CreatedAt: newTimestampFromTime(time.Now()),
-		Files:     []*MFFilePath{},
-	}
-	for _, f := range m.files {
-		nf := &MFFilePath{
-			Path: f.path,
-			// FIXME add more stuff
-		}
-		m.pbInner.Files = append(m.pbInner.Files, nf)
-	}
-	return nil
-}
--- a/modcache.tzst
+++ b/modcache.tzst
--- a/vendor.tzst
+++ b/vendor.tzst
Author	SHA1	Message	Date
sneak	295919fbc9	small progress Some checks failed continuous-integration/drone/push Build is failing Details continuous-integration/drone/pr Build is failing Details	2022-12-04 03:18:47 +04:00
sneak	0f641db567	wtf Some checks failed continuous-integration/drone/push Build is failing Details continuous-integration/drone/pr Build is failing Details	2022-12-02 02:05:09 +01:00
sneak	b8e544bca8	works but has a weird bug Some checks failed continuous-integration/drone/push Build is failing Details continuous-integration/drone/pr Build is failing Details	2022-12-02 01:54:01 +01:00
sneak	5c76824e14	builds now	2022-12-02 01:31:49 +01:00
sneak	4ac80cfcec	vendor updates Some checks failed continuous-integration/drone/push Build is failing Details continuous-integration/drone/pr Build is failing Details	2022-02-02 01:46:51 -08:00
sneak	6ba6eb876b	latest	2022-02-02 01:46:25 -08:00
sneak	591db8d541	bump protoc version in build, use newer syntax in .proto All checks were successful continuous-integration/drone/push Build is passing Details continuous-integration/drone/pr Build is passing Details	2022-02-02 00:36:49 -08:00