Compare commits
No commits in common. "17ad86642a7cc86bc368eca2988933b3be2b7555" and "a2bf7ee607c85b0e6c979837cb7df492e4452b94" have entirely different histories.
17ad86642a
...
a2bf7ee607
@ -1,4 +1,5 @@
|
||||
*.tzst
|
||||
*.tar
|
||||
/buildimage
|
||||
/dockerdeps
|
||||
/tmp
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,8 +1,8 @@
|
||||
mfer/*.pb.go
|
||||
/mfer.cmd
|
||||
vendor
|
||||
/tmp
|
||||
*.tmp
|
||||
*.docker.tzst
|
||||
*.tzst
|
||||
/builddeps/modcache.tar
|
||||
/vendor
|
||||
|
||||
29
Dockerfile
29
Dockerfile
@ -1,11 +1,35 @@
|
||||
################################################################################
|
||||
#2345678911234567892123456789312345678941234567895123456789612345678971234567898
|
||||
################################################################################
|
||||
FROM sneak/builder:14e59af AS builder
|
||||
FROM golang:1.19.3-bullseye AS builder
|
||||
ENV GOPATH /go
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
RUN --mount=type=cache,target=/var/cache/apt \
|
||||
apt update && \
|
||||
apt install -y make zstd unzip && \
|
||||
mkdir /build
|
||||
WORKDIR /tmp
|
||||
# install newer protoc
|
||||
COPY ./builddeps/* ./
|
||||
RUN unzip protoc-*-linux-$(uname -m).zip -d /usr/local && \
|
||||
protoc --version && \
|
||||
mkdir -p /go/pkg/mod && \
|
||||
cd /go/pkg/mod && \
|
||||
tar xvf /tmp/modcache.tar && \
|
||||
cd / && \
|
||||
go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 && \
|
||||
go install -v mvdan.cc/gofumpt@latest && \
|
||||
go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 && \
|
||||
rm -rf /tmp/*
|
||||
WORKDIR /build
|
||||
COPY ./go.mod ./go.sum .
|
||||
RUN \
|
||||
go mod download -x
|
||||
################################################################################
|
||||
#### caching phase done
|
||||
################################################################################
|
||||
WORKDIR /build
|
||||
COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum .
|
||||
COPY ./vendor.tar /build/vendor.tar
|
||||
COPY ./internal ./internal
|
||||
COPY ./bin/gitrev.sh ./bin/gitrev.sh
|
||||
COPY ./mfer ./mfer
|
||||
@ -15,7 +39,6 @@ ARG GITREV unknown
|
||||
RUN \
|
||||
cd mfer && go generate . && cd .. && \
|
||||
GOPACKAGESDEBUG=true golangci-lint run ./... && \
|
||||
tar xf vendor.tar && rm vendor.tar && \
|
||||
make mfer.cmd
|
||||
RUN go mod vendor && tar -c . | zstdmt -19 > /src.tzst
|
||||
################################################################################
|
||||
|
||||
8
Makefile
8
Makefile
@ -58,7 +58,8 @@ lint:
|
||||
|
||||
docker: sneak-mfer.$(ARCH).docker.tzst
|
||||
|
||||
sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES) vendor.tar
|
||||
sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES)
|
||||
bash -x bin/docker-prereqs.sh
|
||||
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
|
||||
docker save sneak/mfer | pv | zstdmt -19 > $@
|
||||
du -sh $@
|
||||
@ -66,8 +67,3 @@ sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES) vendor.tar
|
||||
godoc:
|
||||
open http://127.0.0.1:6060
|
||||
godoc -http=:6060
|
||||
|
||||
vendor.tar: go.mod go.sum
|
||||
go mod vendor
|
||||
tar -c vendor > $@
|
||||
rm -rf vendor
|
||||
|
||||
47
bin/docker-prereqs.sh
Normal file
47
bin/docker-prereqs.sh
Normal file
@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
set -euxo pipefail
|
||||
GOI="golang:1.19.3-bullseye"
|
||||
#CII="golangci/golangci-lint:v1.50.1"
|
||||
|
||||
if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then
|
||||
mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
|
||||
fi
|
||||
|
||||
DICD="$DOCKER_IMAGE_CACHE_DIR"
|
||||
|
||||
function buildImageCache() {
|
||||
if [[ ! -e "$DICD/go.tzst" ]]; then
|
||||
docker pull $GOI
|
||||
docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \
|
||||
mv $DICD/go.tzst.tmp $DICD/go.tzst
|
||||
fi
|
||||
|
||||
#if [[ ! -e "$DICD/ci.tzst" ]]; then
|
||||
# docker pull $CII
|
||||
# docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \
|
||||
# mv $DICD/ci.tzst.tmp $DICD/ci.tzst
|
||||
#fi
|
||||
}
|
||||
|
||||
function loadImageCache() {
|
||||
# zstdmt -d --stdout $DICD/ci.tzst | pv | docker load
|
||||
zstdmt -d --stdout $DICD/go.tzst | pv | docker load
|
||||
|
||||
#docker image ls $CII
|
||||
docker image ls $GOI
|
||||
|
||||
}
|
||||
|
||||
function writeModuleCache() {
|
||||
if [[ ! -e $DICD/modcache.tar ]]; then
|
||||
cd $(go env GOMODCACHE)
|
||||
tar -c . | pv > $DICD/modcache.tar.tmp && mv $DICD/modcache.tar.tmp $DICD/modcache.tar
|
||||
cd -
|
||||
fi
|
||||
cp -av $DICD/modcache.tar ./builddeps/modcache.tar.tmp && mv ./builddeps/modcache.tar.tmp ./builddeps/modcache.tar
|
||||
}
|
||||
|
||||
buildImageCache
|
||||
loadImageCache
|
||||
writeModuleCache
|
||||
|
||||
BIN
builddeps/protoc-21.10-linux-aarch64.zip
Normal file
BIN
builddeps/protoc-21.10-linux-aarch64.zip
Normal file
Binary file not shown.
BIN
builddeps/protoc-21.10-linux-x86_64.zip
Normal file
BIN
builddeps/protoc-21.10-linux-x86_64.zip
Normal file
Binary file not shown.
21
buildimage/Dockerfile
Normal file
21
buildimage/Dockerfile
Normal file
@ -0,0 +1,21 @@
|
||||
## build image:
|
||||
FROM golang:1.19.3-bullseye AS builder
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
RUN apt update && apt install -y make bzip2 curl unzip
|
||||
RUN mkdir -p /build
|
||||
WORKDIR /build
|
||||
|
||||
# install newer protoc
|
||||
RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \
|
||||
unzip *.zip -d /usr/local && rm -v *.zip && protoc --version
|
||||
|
||||
RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
|
||||
|
||||
RUN go env
|
||||
|
||||
COPY ./go.mod .
|
||||
COPY ./go.sum .
|
||||
|
||||
RUN --mount=type=cache,target=/go/pkg go mod download -x
|
||||
RUN rm -rfv /var/cache/* /var/tmp/*
|
||||
1
go.mod
1
go.mod
@ -25,6 +25,7 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
|
||||
|
||||
10
go.sum
10
go.sum
@ -37,7 +37,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
|
||||
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
|
||||
github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
|
||||
@ -62,6 +61,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@ -189,9 +190,13 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
|
||||
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
|
||||
@ -216,6 +221,8 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
|
||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
||||
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
||||
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
|
||||
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
|
||||
github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
|
||||
github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
|
||||
@ -530,6 +537,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@ -1,15 +0,0 @@
|
||||
package bork
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrMissingMagic = errors.New("missing magic bytes in file")
|
||||
ErrFileTruncated = errors.New("file/stream is truncated abnormally")
|
||||
)
|
||||
|
||||
func Newf(format string, args ...interface{}) error {
|
||||
return fmt.Errorf(format, args...)
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
package bork
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestBuild(t *testing.T) {
|
||||
assert.NotNil(t, ErrMissingMagic)
|
||||
}
|
||||
32
internal/cli/misc.go
Normal file
32
internal/cli/misc.go
Normal file
@ -0,0 +1,32 @@
|
||||
package cli
|
||||
|
||||
import "fmt"
|
||||
|
||||
// FIXME make this write to a bytes.Buffer with fprintf
|
||||
func dumpByteSlice(b []byte) {
|
||||
var a [16]byte
|
||||
n := (len(b) + 15) &^ 15
|
||||
for i := 0; i < n; i++ {
|
||||
if i%16 == 0 {
|
||||
fmt.Printf("%4d", i)
|
||||
}
|
||||
if i%8 == 0 {
|
||||
fmt.Print(" ")
|
||||
}
|
||||
if i < len(b) {
|
||||
fmt.Printf(" %02X", b[i])
|
||||
} else {
|
||||
fmt.Print(" ")
|
||||
}
|
||||
if i >= len(b) {
|
||||
a[i%16] = ' '
|
||||
} else if b[i] < 32 || b[i] > 126 {
|
||||
a[i%16] = '.'
|
||||
} else {
|
||||
a[i%16] = b[i]
|
||||
}
|
||||
if i%16 == 15 {
|
||||
fmt.Printf(" %s\n", string(a[:]))
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,9 +1,6 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/apex/log"
|
||||
acli "github.com/apex/log/handlers/cli"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
@ -28,25 +25,13 @@ func Init() {
|
||||
log.SetLevel(log.InfoLevel)
|
||||
}
|
||||
|
||||
func Debugf(format string, args ...interface{}) {
|
||||
DebugReal(fmt.Sprintf(format, args...), 2)
|
||||
}
|
||||
|
||||
func Debug(arg string) {
|
||||
DebugReal(arg, 2)
|
||||
}
|
||||
|
||||
func DebugReal(arg string, cs int) {
|
||||
_, callerFile, callerLine, ok := runtime.Caller(cs)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
tag := fmt.Sprintf("%s:%d: ", callerFile, callerLine)
|
||||
log.Debug(tag + arg)
|
||||
log.Debug(arg)
|
||||
}
|
||||
|
||||
func Dump(args ...interface{}) {
|
||||
DebugReal(spew.Sdump(args...), 2)
|
||||
str := spew.Sdump(args...)
|
||||
Debug(str)
|
||||
}
|
||||
|
||||
func EnableDebugLogging() {
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestBuild(t *testing.T) {
|
||||
Init()
|
||||
assert.True(t, true)
|
||||
}
|
||||
@ -1,89 +0,0 @@
|
||||
package mfer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"git.eeqj.de/sneak/mfer/internal/bork"
|
||||
"git.eeqj.de/sneak/mfer/internal/log"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func (m *manifest) validateProtoOuter() error {
|
||||
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
|
||||
return errors.New("unknown version")
|
||||
}
|
||||
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
|
||||
return errors.New("unknown compression type")
|
||||
}
|
||||
|
||||
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
|
||||
|
||||
gzr, err := gzip.NewReader(bb)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dat, err := io.ReadAll(gzr)
|
||||
defer gzr.Close()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isize := len(dat)
|
||||
if int64(isize) != m.pbOuter.Size {
|
||||
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
|
||||
return bork.ErrFileTruncated
|
||||
}
|
||||
log.Debugf("inner data size is %d", isize)
|
||||
log.Dump(dat)
|
||||
log.Dump(m.pbOuter.Sha256)
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateMagic(dat []byte) bool {
|
||||
ml := len([]byte(MAGIC))
|
||||
if len(dat) < ml {
|
||||
return false
|
||||
}
|
||||
got := dat[0:ml]
|
||||
expected := []byte(MAGIC)
|
||||
return bytes.Equal(got, expected)
|
||||
}
|
||||
|
||||
func NewFromProto(input io.Reader) (*manifest, error) {
|
||||
m := New()
|
||||
dat, err := io.ReadAll(input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !validateMagic(dat) {
|
||||
return nil, errors.New("invalid file format")
|
||||
}
|
||||
|
||||
// remove magic bytes prefix:
|
||||
ml := len([]byte(MAGIC))
|
||||
bb := bytes.NewBuffer(dat[ml:])
|
||||
dat = bb.Bytes()
|
||||
|
||||
log.Dump(dat)
|
||||
|
||||
// deserialize:
|
||||
m.pbOuter = new(MFFileOuter)
|
||||
err = proto.Unmarshal(dat, m.pbOuter)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ve := m.validateProtoOuter()
|
||||
if ve != nil {
|
||||
return nil, ve
|
||||
}
|
||||
|
||||
// FIXME TODO deserialize inner
|
||||
return m, nil
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
package mfer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"git.eeqj.de/sneak/mfer/internal/log"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestAPIExample(t *testing.T) {
|
||||
// read from filesystem
|
||||
m, err := NewFromFS(&ManifestScanOptions{
|
||||
IgnoreDotfiles: true,
|
||||
}, big)
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, m)
|
||||
|
||||
// scan for files
|
||||
m.Scan()
|
||||
|
||||
// serialize
|
||||
var buf bytes.Buffer
|
||||
m.WriteTo(&buf)
|
||||
|
||||
// show serialized
|
||||
log.Dump(buf.Bytes())
|
||||
|
||||
// do it again
|
||||
var buf2 bytes.Buffer
|
||||
m.WriteTo(&buf2)
|
||||
|
||||
// should be same!
|
||||
assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
|
||||
|
||||
// deserialize
|
||||
m2, err := NewFromProto(&buf)
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, m2)
|
||||
|
||||
log.Dump(m2)
|
||||
}
|
||||
@ -65,7 +65,8 @@ func (m *manifest) addInputPath(inputPath string) error {
|
||||
}
|
||||
// FIXME check to make sure inputPath/abs exists maybe
|
||||
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs))
|
||||
return m.addInputFS(afs)
|
||||
m.addInputFS(afs)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *manifest) addInputFS(f afero.Fs) error {
|
||||
@ -158,6 +159,7 @@ func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
|
||||
|
||||
func (m *manifest) Scan() error {
|
||||
// FIXME scan and whatever function does the hashing should take ctx
|
||||
log.Debug("manifest scanning")
|
||||
for idx, sfs := range m.sourceFS {
|
||||
if sfs == nil {
|
||||
return errors.New("invalid source fs")
|
||||
|
||||
@ -9,26 +9,19 @@ message Timestamp {
|
||||
|
||||
message MFFileOuter {
|
||||
enum Version {
|
||||
VERSION_NONE = 0;
|
||||
VERSION_ONE = 1; // only one for now
|
||||
NONE = 0;
|
||||
ONE = 1; // only one for now
|
||||
}
|
||||
|
||||
// required mffile root attributes 1xx
|
||||
Version version = 101;
|
||||
|
||||
enum CompressionType {
|
||||
COMPRESSION_NONE = 0;
|
||||
COMPRESSION_GZIP = 1;
|
||||
}
|
||||
|
||||
CompressionType compressionType = 102;
|
||||
|
||||
bytes innerMessage = 102;
|
||||
// these are used solely to detect corruption/truncation
|
||||
// and not for cryptographic integrity.
|
||||
int64 size = 103;
|
||||
bytes sha256 = 104;
|
||||
bytes deleteme = 105;
|
||||
|
||||
bytes innerMessage = 199;
|
||||
// 2xx for optional manifest root attributes
|
||||
// think we might use gosignify instead of gpg:
|
||||
// github.com/frankbraun/gosignify
|
||||
@ -64,8 +57,8 @@ message MFFileChecksum {
|
||||
|
||||
message MFFile {
|
||||
enum Version {
|
||||
VERSION_NONE = 0;
|
||||
VERSION_ONE = 1; // only one for now
|
||||
NONE = 0;
|
||||
ONE = 1; // only one for now
|
||||
}
|
||||
Version version = 100;
|
||||
|
||||
|
||||
@ -2,10 +2,10 @@ package mfer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"git.eeqj.de/sneak/mfer/internal/log"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
@ -16,25 +16,19 @@ var (
|
||||
)
|
||||
|
||||
var (
|
||||
af *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
|
||||
big *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
|
||||
mf afero.Fs = afero.NewMemMapFs()
|
||||
af *afero.Afero = &afero.Afero{Fs: mf}
|
||||
)
|
||||
|
||||
func init() {
|
||||
log.EnableDebugLogging()
|
||||
|
||||
// create test files and directories
|
||||
af.MkdirAll("/a/b/c", 0o755)
|
||||
af.MkdirAll("/.hidden", 0o755)
|
||||
af.WriteFile("/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
|
||||
af.WriteFile("/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
|
||||
af.WriteFile("/.hidden/hello.txt", []byte("hello world\n"), 0o755)
|
||||
af.WriteFile("/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
|
||||
|
||||
big.MkdirAll("/home/user/Library", 0o755)
|
||||
for i := range [25]int{} {
|
||||
big.WriteFile(fmt.Sprintf("/home/user/Library/hello%d.txt", i), []byte("hello world\n"), 0o755)
|
||||
}
|
||||
afero.WriteFile(af, "/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
|
||||
afero.WriteFile(af, "/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
|
||||
afero.WriteFile(af, "/.hidden/hello.txt", []byte("hello world\n"), 0o755)
|
||||
afero.WriteFile(af, "/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
|
||||
log.EnableDebugLogging()
|
||||
}
|
||||
|
||||
func TestPathHiddenFunc(t *testing.T) {
|
||||
@ -51,7 +45,6 @@ func TestManifestGenerationOne(t *testing.T) {
|
||||
}, af)
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, m)
|
||||
m.Scan()
|
||||
assert.Equal(t, int64(2), m.GetFileCount())
|
||||
assert.Equal(t, int64(30), m.GetTotalFileSize())
|
||||
}
|
||||
@ -62,7 +55,7 @@ func TestManifestGenerationTwo(t *testing.T) {
|
||||
}, af)
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, m)
|
||||
m.Scan()
|
||||
spew.Dump(m)
|
||||
assert.Equal(t, int64(4), m.GetFileCount())
|
||||
assert.Equal(t, int64(54), m.GetTotalFileSize())
|
||||
err = m.generate()
|
||||
@ -70,5 +63,5 @@ func TestManifestGenerationTwo(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
err = m.WriteTo(&buf)
|
||||
assert.Nil(t, err)
|
||||
log.Dump(buf.Bytes())
|
||||
spew.Dump(buf)
|
||||
}
|
||||
|
||||
@ -2,19 +2,15 @@ package mfer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/mfer/internal/log"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
|
||||
|
||||
// rot13("MANIFEST")
|
||||
const MAGIC string = "ZNAVSRFG"
|
||||
|
||||
func newTimestampFromTime(t time.Time) *Timestamp {
|
||||
out := &Timestamp{
|
||||
Seconds: t.Unix(),
|
||||
@ -24,6 +20,10 @@ func newTimestampFromTime(t time.Time) *Timestamp {
|
||||
}
|
||||
|
||||
func (m *manifest) generate() error {
|
||||
log.Debug("generate()")
|
||||
|
||||
const MAGIC string = "ZNAVSRFG"
|
||||
|
||||
if m.pbInner == nil {
|
||||
e := m.generateInner()
|
||||
if e != nil {
|
||||
@ -36,7 +36,7 @@ func (m *manifest) generate() error {
|
||||
return e
|
||||
}
|
||||
}
|
||||
dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
|
||||
dat, err := proto.Marshal(m.pbOuter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -49,46 +49,29 @@ func (m *manifest) generate() error {
|
||||
}
|
||||
|
||||
func (m *manifest) generateOuter() error {
|
||||
log.Debug("generateOuter()")
|
||||
if m.pbInner == nil {
|
||||
return errors.New("internal error")
|
||||
}
|
||||
innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
|
||||
innerData, err := proto.Marshal(m.pbInner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
h.Write(innerData)
|
||||
|
||||
idc := new(bytes.Buffer)
|
||||
gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = gzw.Write(innerData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gzw.Close()
|
||||
|
||||
o := &MFFileOuter{
|
||||
InnerMessage: idc.Bytes(),
|
||||
Size: int64(len(innerData)),
|
||||
Sha256: h.Sum(nil),
|
||||
Version: MFFileOuter_VERSION_ONE,
|
||||
CompressionType: MFFileOuter_COMPRESSION_GZIP,
|
||||
InnerMessage: innerData,
|
||||
}
|
||||
m.pbOuter = o
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *manifest) generateInner() error {
|
||||
log.Debug("generateInner()")
|
||||
m.pbInner = &MFFile{
|
||||
Version: MFFile_VERSION_ONE,
|
||||
Version: MFFile_ONE,
|
||||
CreatedAt: newTimestampFromTime(time.Now()),
|
||||
Files: []*MFFilePath{},
|
||||
}
|
||||
|
||||
for _, f := range m.files {
|
||||
nf := &MFFilePath{
|
||||
Path: f.path,
|
||||
@ -96,5 +79,6 @@ func (m *manifest) generateInner() error {
|
||||
}
|
||||
m.pbInner.Files = append(m.pbInner.Files, nf)
|
||||
}
|
||||
log.Dump(m.pbInner)
|
||||
return nil
|
||||
}
|
||||
|
||||
BIN
vendor.tar
BIN
vendor.tar
Binary file not shown.
Loading…
Reference in New Issue
Block a user