sneak
/
mfer
1
0
Fork 0
Code Issues 3 Pull Requests 2 Releases Wiki Activity

next #5

Merged
sneak merged 14 commits from next into master 2022-12-09 00:02:35 +00:00
Conversation 0 Commits 14 Files Changed 29 +54 -91
13 changed files with 54 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit aa3c159521 - Show all commits

19
Dockerfile
View File

@ -1,26 +1,9 @@
################################################################################
#2345678911234567892123456789312345678941234567895123456789612345678971234567898
################################################################################
FROM golang:1.19.3-bullseye AS builder
FROM sneak/builder:main AS builder
ENV GOPATH /go
ENV DEBIAN_FRONTEND noninteractive
RUN --mount=type=cache,target=/var/cache/apt \
apt update && \
apt install -y make zstd unzip && \
mkdir /build
WORKDIR /tmp
# install newer protoc
COPY ./builddeps/* ./
RUN unzip protoc-*-linux-$(uname -m).zip -d /usr/local && \
protoc --version && \
mkdir -p /go/pkg/mod && \
cd /go/pkg/mod && \
tar xvf /tmp/modcache.tar && \
cd / && \
go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 && \
go install -v mvdan.cc/gofumpt@latest && \
go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 && \
rm -rf /tmp/*
WORKDIR /build
COPY ./go.mod ./go.sum .
RUN \

1
Makefile
View File

@ -59,7 +59,6 @@ lint:
docker: sneak-mfer.$(ARCH).docker.tzst
sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES)
bash -x bin/docker-prereqs.sh
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
docker save sneak/mfer | pv | zstdmt -19 > $@
du -sh $@

47
bin/docker-prereqs.sh
View File

@ -1,47 +0,0 @@
#!/bin/bash
set -euxo pipefail
GOI="golang:1.19.3-bullseye"
#CII="golangci/golangci-lint:v1.50.1"
if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then
mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
fi
DICD="$DOCKER_IMAGE_CACHE_DIR"
function buildImageCache() {
if [[ ! -e "$DICD/go.tzst" ]]; then
docker pull $GOI
docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \
mv $DICD/go.tzst.tmp $DICD/go.tzst
fi
#if [[ ! -e "$DICD/ci.tzst" ]]; then
# docker pull $CII
# docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \
# mv $DICD/ci.tzst.tmp $DICD/ci.tzst
#fi
}
function loadImageCache() {
# zstdmt -d --stdout $DICD/ci.tzst | pv | docker load
zstdmt -d --stdout $DICD/go.tzst | pv | docker load
#docker image ls $CII
docker image ls $GOI
}
function writeModuleCache() {
if [[ ! -e $DICD/modcache.tar ]]; then
cd $(go env GOMODCACHE)
tar -c . | pv > $DICD/modcache.tar.tmp && mv $DICD/modcache.tar.tmp $DICD/modcache.tar
cd -
fi
cp -av $DICD/modcache.tar ./builddeps/modcache.tar.tmp && mv ./builddeps/modcache.tar.tmp ./builddeps/modcache.tar
}
buildImageCache
loadImageCache
writeModuleCache

BIN
builddeps/protoc-21.10-linux-aarch64.zip
View File

Binary file not shown.

BIN
builddeps/protoc-21.10-linux-x86_64.zip
View File

Binary file not shown.

1
go.mod
View File

@ -25,7 +25,6 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect

10
go.sum
View File

@ -37,6 +37,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
@ -61,8 +62,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -190,13 +189,9 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
@ -221,8 +216,6 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
@ -537,7 +530,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

2
internal/cli/misc.go
View File

@ -3,7 +3,7 @@ package cli
import "fmt"
// FIXME make this write to a bytes.Buffer with fprintf
func dumpByteSlice(b []byte) {
func DumpByteSlice(b []byte) {
var a [16]byte
n := (len(b) + 15) &^ 15
for i := 0; i < n; i++ {

20
mfer/example_test.go Normal file
View File

@ -0,0 +1,20 @@
package mfer
import (
"bytes"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/stretchr/testify/assert"
)
func TestAPIExample(t *testing.T) {
m, err := NewFromFS(&ManifestScanOptions{
IgnoreDotfiles: true,
}, af)
assert.NotNil(t, err)
m.Scan()
var buf bytes.Buffer
m.WriteTo(&buf)
spew.Dump(buf.Bytes())
}

5
mfer/manifest.go
View File

@ -65,8 +65,7 @@ func (m *manifest) addInputPath(inputPath string) error {
}
// FIXME check to make sure inputPath/abs exists maybe
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs))
m.addInputFS(afs)
return nil
return m.addInputFS(afs)
}
func (m *manifest) addInputFS(f afero.Fs) error {
@ -159,7 +158,7 @@ func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
func (m *manifest) Scan() error {
// FIXME scan and whatever function does the hashing should take ctx
log.Debug("manifest scanning")
log.Debug("manifest Scan()")
for idx, sfs := range m.sourceFS {
if sfs == nil {
return errors.New("invalid source fs")

19
mfer/mf.proto
View File

@ -9,19 +9,26 @@ message Timestamp {
message MFFileOuter {
enum Version {
NONE = 0;
ONE = 1; // only one for now
VERSION_NONE = 0;
VERSION_ONE = 1; // only one for now
}
// required mffile root attributes 1xx
Version version = 101;
bytes innerMessage = 102;
enum CompressionType {
COMPRESSION_NONE = 0;
COMPRESSION_GZIP = 1;
}
CompressionType compressionType = 102;
// these are used solely to detect corruption/truncation
// and not for cryptographic integrity.
int64 size = 103;
bytes sha256 = 104;
bytes deleteme = 105;
bytes innerMessage = 199;
// 2xx for optional manifest root attributes
// think we might use gosignify instead of gpg:
// github.com/frankbraun/gosignify
@ -57,8 +64,8 @@ message MFFileChecksum {
message MFFile {
enum Version {
NONE = 0;
ONE = 1; // only one for now
VERSION_NONE = 0;
VERSION_ONE = 1; // only one for now
}
Version version = 100;

6
mfer/mfer_test.go
View File

@ -5,7 +5,6 @@ import (
"testing"
"git.eeqj.de/sneak/mfer/internal/log"
"github.com/davecgh/go-spew/spew"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
)
@ -45,6 +44,7 @@ func TestManifestGenerationOne(t *testing.T) {
}, af)
assert.Nil(t, err)
assert.NotNil(t, m)
m.Scan()
assert.Equal(t, int64(2), m.GetFileCount())
assert.Equal(t, int64(30), m.GetTotalFileSize())
}
@ -55,7 +55,7 @@ func TestManifestGenerationTwo(t *testing.T) {
}, af)
assert.Nil(t, err)
assert.NotNil(t, m)
spew.Dump(m)
m.Scan()
assert.Equal(t, int64(4), m.GetFileCount())
assert.Equal(t, int64(54), m.GetTotalFileSize())
err = m.generate()
@ -63,5 +63,5 @@ func TestManifestGenerationTwo(t *testing.T) {
var buf bytes.Buffer
err = m.WriteTo(&buf)
assert.Nil(t, err)
spew.Dump(buf)
log.Dump(buf.Bytes())
}

15
mfer/serialize.go
View File

@ -2,6 +2,7 @@ package mfer
import (
"bytes"
"crypto/sha256"
"errors"
"time"
@ -57,17 +58,27 @@ func (m *manifest) generateOuter() error {
if err != nil {
return err
}
h := sha256.New()
h.Write(innerData)
o := &MFFileOuter{
InnerMessage: innerData,
// FIXME add more
InnerMessage: innerData,
Size: int64(len(innerData)),
Sha256: h.Sum(nil),
Version: MFFileOuter_VERSION_ONE,
CompressionType: MFFileOuter_COMPRESSION_GZIP,
}
m.pbOuter = o
log.Dump(m.pbOuter)
return nil
}
func (m *manifest) generateInner() error {
log.Debug("generateInner()")
m.pbInner = &MFFile{
Version: MFFile_ONE,
Version: MFFile_VERSION_ONE,
CreatedAt: newTimestampFromTime(time.Now()),
Files: []*MFFilePath{},
}