diff --git a/Dockerfile b/Dockerfile index 436a751..501c442 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,9 @@ ################################################################################ #2345678911234567892123456789312345678941234567895123456789612345678971234567898 ################################################################################ -FROM golang:1.19.3-bullseye AS builder +FROM sneak/builder:main AS builder ENV GOPATH /go ENV DEBIAN_FRONTEND noninteractive -RUN --mount=type=cache,target=/var/cache/apt \ - apt update && \ - apt install -y make zstd unzip && \ - mkdir /build -WORKDIR /tmp -# install newer protoc -COPY ./builddeps/* ./ -RUN unzip protoc-*-linux-$(uname -m).zip -d /usr/local && \ - protoc --version && \ - mkdir -p /go/pkg/mod && \ - cd /go/pkg/mod && \ - tar xvf /tmp/modcache.tar && \ - cd / && \ - go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 && \ - go install -v mvdan.cc/gofumpt@latest && \ - go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 && \ - rm -rf /tmp/* WORKDIR /build COPY ./go.mod ./go.sum . RUN \ diff --git a/Makefile b/Makefile index 24e9a00..76300a7 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,6 @@ lint: docker: sneak-mfer.$(ARCH).docker.tzst sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES) - bash -x bin/docker-prereqs.sh docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer . docker save sneak/mfer | pv | zstdmt -19 > $@ du -sh $@ diff --git a/bin/docker-prereqs.sh b/bin/docker-prereqs.sh deleted file mode 100644 index 40a781f..0000000 --- a/bin/docker-prereqs.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -euxo pipefail -GOI="golang:1.19.3-bullseye" -#CII="golangci/golangci-lint:v1.50.1" - -if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then - mkdir -p "$DOCKER_IMAGE_CACHE_DIR" -fi - -DICD="$DOCKER_IMAGE_CACHE_DIR" - -function buildImageCache() { - if [[ ! -e "$DICD/go.tzst" ]]; then - docker pull $GOI - docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \ - mv $DICD/go.tzst.tmp $DICD/go.tzst - fi - - #if [[ ! -e "$DICD/ci.tzst" ]]; then - # docker pull $CII - # docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \ - # mv $DICD/ci.tzst.tmp $DICD/ci.tzst - #fi -} - -function loadImageCache() { - # zstdmt -d --stdout $DICD/ci.tzst | pv | docker load - zstdmt -d --stdout $DICD/go.tzst | pv | docker load - - #docker image ls $CII - docker image ls $GOI - -} - -function writeModuleCache() { - if [[ ! -e $DICD/modcache.tar ]]; then - cd $(go env GOMODCACHE) - tar -c . | pv > $DICD/modcache.tar.tmp && mv $DICD/modcache.tar.tmp $DICD/modcache.tar - cd - - fi - cp -av $DICD/modcache.tar ./builddeps/modcache.tar.tmp && mv ./builddeps/modcache.tar.tmp ./builddeps/modcache.tar -} - -buildImageCache -loadImageCache -writeModuleCache - diff --git a/builddeps/protoc-21.10-linux-aarch64.zip b/builddeps/protoc-21.10-linux-aarch64.zip deleted file mode 100644 index 14a83d7..0000000 Binary files a/builddeps/protoc-21.10-linux-aarch64.zip and /dev/null differ diff --git a/builddeps/protoc-21.10-linux-x86_64.zip b/builddeps/protoc-21.10-linux-x86_64.zip deleted file mode 100644 index 462d2a3..0000000 Binary files a/builddeps/protoc-21.10-linux-x86_64.zip and /dev/null differ diff --git a/go.mod b/go.mod index c8a7ebb..2d4dcff 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect diff --git a/go.sum b/go.sum index f82af82..0a2d55c 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= @@ -61,8 +62,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -190,13 +189,9 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= @@ -221,8 +216,6 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4= github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= @@ -537,7 +530,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cli/misc.go b/internal/cli/misc.go index f017f8a..b4bc086 100644 --- a/internal/cli/misc.go +++ b/internal/cli/misc.go @@ -3,7 +3,7 @@ package cli import "fmt" // FIXME make this write to a bytes.Buffer with fprintf -func dumpByteSlice(b []byte) { +func DumpByteSlice(b []byte) { var a [16]byte n := (len(b) + 15) &^ 15 for i := 0; i < n; i++ { diff --git a/mfer/example_test.go b/mfer/example_test.go new file mode 100644 index 0000000..678cd23 --- /dev/null +++ b/mfer/example_test.go @@ -0,0 +1,20 @@ +package mfer + +import ( + "bytes" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/assert" +) + +func TestAPIExample(t *testing.T) { + m, err := NewFromFS(&ManifestScanOptions{ + IgnoreDotfiles: true, + }, af) + assert.NotNil(t, err) + m.Scan() + var buf bytes.Buffer + m.WriteTo(&buf) + spew.Dump(buf.Bytes()) +} diff --git a/mfer/manifest.go b/mfer/manifest.go index 2fde3e1..9c26a93 100644 --- a/mfer/manifest.go +++ b/mfer/manifest.go @@ -65,8 +65,7 @@ func (m *manifest) addInputPath(inputPath string) error { } // FIXME check to make sure inputPath/abs exists maybe afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs)) - m.addInputFS(afs) - return nil + return m.addInputFS(afs) } func (m *manifest) addInputFS(f afero.Fs) error { @@ -159,7 +158,7 @@ func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error { func (m *manifest) Scan() error { // FIXME scan and whatever function does the hashing should take ctx - log.Debug("manifest scanning") + log.Debug("manifest Scan()") for idx, sfs := range m.sourceFS { if sfs == nil { return errors.New("invalid source fs") diff --git a/mfer/mf.proto b/mfer/mf.proto index 5e350a1..ebac757 100644 --- a/mfer/mf.proto +++ b/mfer/mf.proto @@ -9,19 +9,26 @@ message Timestamp { message MFFileOuter { enum Version { - NONE = 0; - ONE = 1; // only one for now + VERSION_NONE = 0; + VERSION_ONE = 1; // only one for now } // required mffile root attributes 1xx Version version = 101; - bytes innerMessage = 102; + + enum CompressionType { + COMPRESSION_NONE = 0; + COMPRESSION_GZIP = 1; + } + + CompressionType compressionType = 102; + // these are used solely to detect corruption/truncation // and not for cryptographic integrity. int64 size = 103; bytes sha256 = 104; - bytes deleteme = 105; + bytes innerMessage = 199; // 2xx for optional manifest root attributes // think we might use gosignify instead of gpg: // github.com/frankbraun/gosignify @@ -57,8 +64,8 @@ message MFFileChecksum { message MFFile { enum Version { - NONE = 0; - ONE = 1; // only one for now + VERSION_NONE = 0; + VERSION_ONE = 1; // only one for now } Version version = 100; diff --git a/mfer/mfer_test.go b/mfer/mfer_test.go index d9ec855..2b01455 100644 --- a/mfer/mfer_test.go +++ b/mfer/mfer_test.go @@ -5,7 +5,6 @@ import ( "testing" "git.eeqj.de/sneak/mfer/internal/log" - "github.com/davecgh/go-spew/spew" "github.com/spf13/afero" "github.com/stretchr/testify/assert" ) @@ -45,6 +44,7 @@ func TestManifestGenerationOne(t *testing.T) { }, af) assert.Nil(t, err) assert.NotNil(t, m) + m.Scan() assert.Equal(t, int64(2), m.GetFileCount()) assert.Equal(t, int64(30), m.GetTotalFileSize()) } @@ -55,7 +55,7 @@ func TestManifestGenerationTwo(t *testing.T) { }, af) assert.Nil(t, err) assert.NotNil(t, m) - spew.Dump(m) + m.Scan() assert.Equal(t, int64(4), m.GetFileCount()) assert.Equal(t, int64(54), m.GetTotalFileSize()) err = m.generate() @@ -63,5 +63,5 @@ func TestManifestGenerationTwo(t *testing.T) { var buf bytes.Buffer err = m.WriteTo(&buf) assert.Nil(t, err) - spew.Dump(buf) + log.Dump(buf.Bytes()) } diff --git a/mfer/serialize.go b/mfer/serialize.go index edac9ad..ba6dbe9 100644 --- a/mfer/serialize.go +++ b/mfer/serialize.go @@ -2,6 +2,7 @@ package mfer import ( "bytes" + "crypto/sha256" "errors" "time" @@ -57,17 +58,27 @@ func (m *manifest) generateOuter() error { if err != nil { return err } + + h := sha256.New() + h.Write(innerData) + o := &MFFileOuter{ - InnerMessage: innerData, + // FIXME add more + InnerMessage: innerData, + Size: int64(len(innerData)), + Sha256: h.Sum(nil), + Version: MFFileOuter_VERSION_ONE, + CompressionType: MFFileOuter_COMPRESSION_GZIP, } m.pbOuter = o + log.Dump(m.pbOuter) return nil } func (m *manifest) generateInner() error { log.Debug("generateInner()") m.pbInner = &MFFile{ - Version: MFFile_ONE, + Version: MFFile_VERSION_ONE, CreatedAt: newTimestampFromTime(time.Now()), Files: []*MFFilePath{}, }