.dockerignore

@@ -0,0 +1,5 @@
+*.tzst
+*.tar
+/buildimage
+/dockerdeps
+/tmp

.gitignore

@@ -1,2 +1,6 @@
 mfer/*.pb.go
 /mfer.cmd
+vendor
+/tmp
+/buildimage/go.mod
+/buildimage/go.sum

Dockerfile

@@ -1,6 +1,5 @@
 ## lint image
-## current as of 2022-01-25
+FROM golangci/golangci-lint:v1.50.1
-FROM golangci/golangci-lint:latest AS linter
 
 RUN mkdir -p /build
 WORKDIR /build
@@ -8,19 +7,7 @@ COPY ./ ./
 RUN golangci-lint run
 
 ## build image:
-# this is golang:1.17.6-buster as of 2022-01-27
+FROM sneak/mfer-build AS builder
-FROM golang@sha256:52a48e0239f4d645b20ac268a60361703afe7feb2df5697fa89f72052cb87a3e AS builder
-#FROM golang:1.16-buster AS builder
-
-ENV DEBIAN_FRONTEND noninteractive
-RUN apt update && apt install -y make bzip2 curl unzip
-
-# install newer protoc than what comes with buster
-ENV PB_REL https://github.com/protocolbuffers/protobuf/releases
-RUN curl -LO $PB_REL/download/v3.19.0/protoc-3.19.0-linux-x86_64.zip && \
-    unzip protoc-3.19.0-linux-x86_64.zip -d /usr/local
-
-RUN mkdir -p /build
 WORKDIR /build
 
 COPY go.mod .

Makefile

@@ -1,22 +1,24 @@
-
+export DOCKER_BUILDKIT := 1
 GOPATH := $(shell go env GOPATH)
 export PATH := $(PATH):$(GOPATH)/bin
 PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
 
 ARCH := $(shell uname -m)
 GITREV := $(shell git describe --always --dirty=-dirty)
+APPNAME := mfer
+export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
 
 GOLDFLAGS += -X main.Version=0.1.0
 GOLDFLAGS += -X main.Gitrev=$(GITREV)
 GOFLAGS := -ldflags "$(GOLDFLAGS)"
 
-default: run
+default: test
 
 run: ./mfer.cmd
 	./$<
 	./$< gen --ignore-dotfiles
 
-test:
+test: fmt
 	go test -v --timeout 3s ./...
 
 $(PROTOC_GEN_GO):
@@ -29,7 +31,7 @@ devprereqs:
 	which gofumpt || go install -v mvdan.cc/gofumpt@latest
 	which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest
 
-mfer.cmd: $(PROTOC_GEN_GO) mfer/*.go internal/*/*.go cmd/*/*.go
+mfer.cmd: mfer/*.go internal/*/*.go cmd/*/*.go
 	protoc --version
 	cd mfer && go generate .
 	make test
@@ -48,5 +50,10 @@ lint:
 	golangci-lint run
 	sh -c 'test -z "$$(gofmt -l .)"'
 
-dockerbuild:
+docker:
+	bash -x bin/docker-prereqs.sh
 	docker build .
+
+godoc:
+	open http://127.0.0.1:6060
+	godoc -http=:6060

bin/docker-prereqs.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -euxo pipefail
+GOI="golang:1.19.3-bullseye"
+CII="golangci/golangci-lint:v1.50.1"
+BII="sneak/mfer-build"
+
+if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then
+    mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
+fi
+
+DICD="$DOCKER_IMAGE_CACHE_DIR"
+
+function buildImageCache() {
+    if [[ ! -e "$DICD/go.tzst" ]]; then
+        docker pull $GOI
+        docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \
+            mv $DICD/go.tzst.tmp $DICD/go.tzst
+    fi
+
+    if [[ ! -e "$DICD/ci.tzst" ]]; then
+        docker pull $CII
+        docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \
+            mv $DICD/ci.tzst.tmp $DICD/ci.tzst
+    fi
+
+    if [[ ! -e "$DICD/build.tzst" ]]; then
+        go mod download -x
+        cd buildimage
+        cp ../go.mod ../go.sum .
+        docker build -t $BII . && rm -rf go.mod go.sum && \
+        cd .. && \
+        docker save $BII | pv | zstdmt -19 > $DICD/build.tzst.tmp && \
+            mv $DICD/build.tzst.tmp $DICD/build.tzst
+    fi
+}
+
+function loadImageCache() {
+    docker image ls $CII || \
+        zstdmt -d --stdout $DICD/ci.tzst | pv | docker load
+
+    docker image ls $GOI || \
+        zstdmt -d --stdout $DICD/go.tzst | pv | docker load
+
+    docker image ls $BII || \
+        zstdmt -d --stdout $DICD/build.tzst | pv | docker load
+}
+
+buildImageCache
+
+loadImageCache

buildimage/Dockerfile

@@ -0,0 +1,21 @@
+## build image:
+FROM golang:1.19.3-bullseye AS builder
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt update && apt install -y make bzip2 curl unzip
+RUN mkdir -p /build
+WORKDIR /build
+
+# install newer protoc
+RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \
+    unzip *.zip -d /usr/local && rm -v *.zip && protoc --version
+
+RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
+
+RUN go env
+
+COPY ./go.mod .
+COPY ./go.sum .
+
+RUN --mount=type=cache,target=/go/pkg go mod download -x
+RUN rm -rfv /var/cache/* /var/tmp/*

go.mod

@@ -3,11 +3,11 @@ module git.eeqj.de/sneak/mfer
 go 1.16
 
 require (
-	github.com/apex/log v1.9.0 // indirect
+	github.com/apex/log v1.9.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/pterm/pterm v0.12.35
 	github.com/spf13/afero v1.8.0
+	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.3.0
-	github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 // indirect
+	google.golang.org/protobuf v1.28.1
-	google.golang.org/protobuf v1.27.1
 )

go.sum

@@ -154,8 +154,10 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
@@ -197,13 +199,19 @@ github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4S
 github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60=
 github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
+github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
 github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
 github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
 github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
@@ -211,8 +219,6 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
-github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 h1:RDJ3ggSqBL4Mu/ANRKmxuLrwnupJsvHRDXe4/KI+HWE=
-github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20/go.mod h1:iljxuLc3m07jsXOWojqehoTp/Fh0XP7irbhV+6sYNGc=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -512,10 +518,11 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@@ -525,8 +532,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

mfer/manifest.go

@@ -10,8 +10,6 @@ import (
 	"github.com/spf13/afero"
 )
 
-//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
-
 type ManifestFile struct {
 	Path     string
 	FileInfo fs.FileInfo
@@ -27,6 +25,8 @@ type Manifest struct {
 	Files         []*ManifestFile
 	ScanOptions   *ManifestScanOptions
 	TotalFileSize int64
+	PBInner       *MFFile
+	PBOuter       *MFFileOuter
 }
 
 func (m *Manifest) String() string {
@@ -89,6 +89,12 @@ func pathIsHidden(p string) bool {
 	}
 }
 
+/*
+func timeToTimestamp(t time.Time) *Timestamp {
+
+}
+*/
+
 func (m *Manifest) Scan() error {
 	// FIXME scan and whatever function does the hashing should take ctx
 	oe := afero.Walk(m.SourceFS, "/", func(p string, info fs.FileInfo, err error) error {

mfer/mf.proto

@@ -7,7 +7,7 @@ message Timestamp {
   int32 nanos = 2;
 }
 
-message MFFile {
+message MFFileOuter {
     enum Version {
         NONE = 0;
         ONE = 1; // only one for now
@@ -54,7 +54,7 @@ message MFFileChecksum {
     bytes multiHash = 1;
 }
 
-message MFFileInner {
+message MFFile {
     enum Version {
         NONE = 0;
         ONE = 1; // only one for now

mfer/serialize.go

@@ -0,0 +1,12 @@
+package mfer
+
+//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
+
+func (m *Manifest) Generate() error {
+	m.PBInner = &MFFile{
+		Version: MFFile_ONE,
+		// CreatedAt: time.Now(),
+		Files: []*MFFilePath{},
+	}
+	return nil
+}