sneak/mfer
1
0
Fork 0
Code Issues 3 Pull Requests 2 Releases Wiki Activity

latest - uses custom build image now

Browse Source
This commit is contained in:
Jeffrey Paul 2022-12-06 17:43:07 +04:00
parent a2bf7ee607
commit aa3c159521
13 changed files with 54 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
Dockerfile
View File

@ -1,26 +1,9 @@
################################################################################
#2345678911234567892123456789312345678941234567895123456789612345678971234567898
################################################################################
FROM golang:1.19.3-bullseye AS builder
FROM sneak/builder:main AS builder
ENV GOPATH /go
ENV DEBIAN_FRONTEND noninteractive
RUN --mount=type=cache,target=/var/cache/apt \
apt update && \
apt install -y make zstd unzip && \
mkdir /build
WORKDIR /tmp
# install newer protoc
COPY ./builddeps/* ./
RUN unzip protoc-*-linux-$(uname -m).zip -d /usr/local && \
protoc --version && \
mkdir -p /go/pkg/mod && \
cd /go/pkg/mod && \
tar xvf /tmp/modcache.tar && \
cd / && \
go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 && \
go install -v mvdan.cc/gofumpt@latest && \
go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 && \
rm -rf /tmp/*
WORKDIR /build
COPY ./go.mod ./go.sum .
RUN \

1
Makefile
View File

@ -59,7 +59,6 @@ lint:
docker: sneak-mfer.$(ARCH).docker.tzst
sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES)
bash -x bin/docker-prereqs.sh
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
docker save sneak/mfer | pv | zstdmt -19 > $@
du -sh $@

47
bin/docker-prereqs.sh
View File

@ -1,47 +0,0 @@
#!/bin/bash
set -euxo pipefail
GOI="golang:1.19.3-bullseye"
#CII="golangci/golangci-lint:v1.50.1"
if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then
mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
fi
DICD="$DOCKER_IMAGE_CACHE_DIR"
function buildImageCache() {
if [[ ! -e "$DICD/go.tzst" ]]; then
docker pull $GOI
docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \
mv $DICD/go.tzst.tmp $DICD/go.tzst
fi
#if [[ ! -e "$DICD/ci.tzst" ]]; then
# docker pull $CII
# docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \
# mv $DICD/ci.tzst.tmp $DICD/ci.tzst
#fi
}
function loadImageCache() {
# zstdmt -d --stdout $DICD/ci.tzst | pv | docker load
zstdmt -d --stdout $DICD/go.tzst | pv | docker load
#docker image ls $CII
docker image ls $GOI
}
function writeModuleCache() {
if [[ ! -e $DICD/modcache.tar ]]; then
cd $(go env GOMODCACHE)
tar -c . | pv > $DICD/modcache.tar.tmp && mv $DICD/modcache.tar.tmp $DICD/modcache.tar
cd -
fi
cp -av $DICD/modcache.tar ./builddeps/modcache.tar.tmp && mv ./builddeps/modcache.tar.tmp ./builddeps/modcache.tar
}
buildImageCache
loadImageCache
writeModuleCache

BIN
builddeps/protoc-21.10-linux-aarch64.zip
View File

Binary file not shown.

BIN
builddeps/protoc-21.10-linux-x86_64.zip
View File

Binary file not shown.

1
go.mod
View File

@ -25,7 +25,6 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect

10
go.sum
View File

@ -37,6 +37,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
@ -61,8 +62,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -190,13 +189,9 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
@ -221,8 +216,6 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
@ -537,7 +530,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

2
internal/cli/misc.go
View File

@ -3,7 +3,7 @@ package cli
import "fmt"
// FIXME make this write to a bytes.Buffer with fprintf
func dumpByteSlice(b []byte) {
func DumpByteSlice(b []byte) {
var a [16]byte
n := (len(b) + 15) &^ 15
for i := 0; i < n; i++ {

20
mfer/example_test.go Normal file
View File

@ -0,0 +1,20 @@
package mfer
import (
"bytes"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/stretchr/testify/assert"
)
func TestAPIExample(t *testing.T) {
m, err := NewFromFS(&ManifestScanOptions{
IgnoreDotfiles: true,
}, af)
assert.NotNil(t, err)
m.Scan()
var buf bytes.Buffer
m.WriteTo(&buf)
spew.Dump(buf.Bytes())
}

5
mfer/manifest.go
View File

@ -65,8 +65,7 @@ func (m *manifest) addInputPath(inputPath string) error {
}
// FIXME check to make sure inputPath/abs exists maybe
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs))
m.addInputFS(afs)
return nil
return m.addInputFS(afs)
}
func (m *manifest) addInputFS(f afero.Fs) error {
@ -159,7 +158,7 @@ func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
func (m *manifest) Scan() error {
// FIXME scan and whatever function does the hashing should take ctx
log.Debug("manifest scanning")
log.Debug("manifest Scan()")
for idx, sfs := range m.sourceFS {
if sfs == nil {
return errors.New("invalid source fs")

19
mfer/mf.proto
View File

@ -9,19 +9,26 @@ message Timestamp {
message MFFileOuter {
enum Version {
NONE = 0;
ONE = 1; // only one for now
VERSION_NONE = 0;
VERSION_ONE = 1; // only one for now
}
// required mffile root attributes 1xx
Version version = 101;
bytes innerMessage = 102;
enum CompressionType {
COMPRESSION_NONE = 0;
COMPRESSION_GZIP = 1;
}
CompressionType compressionType = 102;
// these are used solely to detect corruption/truncation
// and not for cryptographic integrity.
int64 size = 103;
bytes sha256 = 104;
bytes deleteme = 105;
bytes innerMessage = 199;
// 2xx for optional manifest root attributes
// think we might use gosignify instead of gpg:
// github.com/frankbraun/gosignify
@ -57,8 +64,8 @@ message MFFileChecksum {
message MFFile {
enum Version {
NONE = 0;
ONE = 1; // only one for now
VERSION_NONE = 0;
VERSION_ONE = 1; // only one for now
}
Version version = 100;

6
mfer/mfer_test.go
View File

@ -5,7 +5,6 @@ import (
"testing"
"git.eeqj.de/sneak/mfer/internal/log"
"github.com/davecgh/go-spew/spew"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
)
@ -45,6 +44,7 @@ func TestManifestGenerationOne(t *testing.T) {
}, af)
assert.Nil(t, err)
assert.NotNil(t, m)
m.Scan()
assert.Equal(t, int64(2), m.GetFileCount())
assert.Equal(t, int64(30), m.GetTotalFileSize())
}
@ -55,7 +55,7 @@ func TestManifestGenerationTwo(t *testing.T) {
}, af)
assert.Nil(t, err)
assert.NotNil(t, m)
spew.Dump(m)
m.Scan()
assert.Equal(t, int64(4), m.GetFileCount())
assert.Equal(t, int64(54), m.GetTotalFileSize())
err = m.generate()
@ -63,5 +63,5 @@ func TestManifestGenerationTwo(t *testing.T) {
var buf bytes.Buffer
err = m.WriteTo(&buf)
assert.Nil(t, err)
spew.Dump(buf)
log.Dump(buf.Bytes())
}

13
mfer/serialize.go
View File

@ -2,6 +2,7 @@ package mfer
import (
"bytes"
"crypto/sha256"
"errors"
"time"
@ -57,17 +58,27 @@ func (m *manifest) generateOuter() error {
if err != nil {
return err
}
h := sha256.New()
h.Write(innerData)
o := &MFFileOuter{
// FIXME add more
InnerMessage: innerData,
Size: int64(len(innerData)),
Sha256: h.Sum(nil),
Version: MFFileOuter_VERSION_ONE,
CompressionType: MFFileOuter_COMPRESSION_GZIP,
}
m.pbOuter = o
log.Dump(m.pbOuter)
return nil
}
func (m *manifest) generateInner() error {
log.Debug("generateInner()")
m.pbInner = &MFFile{
Version: MFFile_ONE,
Version: MFFile_VERSION_ONE,
CreatedAt: newTimestampFromTime(time.Now()),
Files: []*MFFilePath{},
}