sneak/mfer
1
0
Fork 0
Code Issues 4 Pull Requests 2 Releases Wiki Activity

Compare commits

base: sneak:17ad86642a7cc86bc368eca2988933b3be2b7555
Branches Tags
sneak:main
sneak:sneak-patch-1
sneak:readme
sneak:next
..
compare: sneak:a2bf7ee607c85b0e6c979837cb7df492e4452b94
Branches Tags
sneak:sneak-patch-1
sneak:main
sneak:readme
sneak:next

No commits in common. "17ad86642a7cc86bc368eca2988933b3be2b7555" and "a2bf7ee607c85b0e6c979837cb7df492e4452b94" have entirely different histories.
17ad86642a ... a2bf7ee607

22 changed files with 175 additions and 258 deletions
Show Stats Expand all files Collapse all files

1
.dockerignore
View File

@ -1,4 +1,5 @@
*.tzst
*.tar
/buildimage
/dockerdeps
/tmp

2
.gitignore vendored
View File

@ -1,8 +1,8 @@
mfer/*.pb.go
/mfer.cmd
vendor
/tmp
*.tmp
*.docker.tzst
*.tzst
/builddeps/modcache.tar
/vendor

29
Dockerfile
View File

@ -1,11 +1,35 @@
################################################################################
#2345678911234567892123456789312345678941234567895123456789612345678971234567898
################################################################################
FROM sneak/builder:14e59af AS builder
FROM golang:1.19.3-bullseye AS builder
ENV GOPATH /go
ENV DEBIAN_FRONTEND noninteractive
RUN --mount=type=cache,target=/var/cache/apt \
apt update && \
apt install -y make zstd unzip && \
mkdir /build
WORKDIR /tmp
# install newer protoc
COPY ./builddeps/* ./
RUN unzip protoc-*-linux-$(uname -m).zip -d /usr/local && \
protoc --version && \
mkdir -p /go/pkg/mod && \
cd /go/pkg/mod && \
tar xvf /tmp/modcache.tar && \
cd / && \
go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 && \
go install -v mvdan.cc/gofumpt@latest && \
go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 && \
rm -rf /tmp/*
WORKDIR /build
COPY ./go.mod ./go.sum .
RUN \
go mod download -x
################################################################################
#### caching phase done
################################################################################
WORKDIR /build
COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum .
COPY ./vendor.tar /build/vendor.tar
COPY ./internal ./internal
COPY ./bin/gitrev.sh ./bin/gitrev.sh
COPY ./mfer ./mfer
@ -15,7 +39,6 @@ ARG GITREV unknown
RUN \
cd mfer && go generate . && cd .. && \
GOPACKAGESDEBUG=true golangci-lint run ./... && \
tar xf vendor.tar && rm vendor.tar && \
make mfer.cmd
RUN go mod vendor && tar -c . | zstdmt -19 > /src.tzst
################################################################################

8
Makefile
View File

@ -58,7 +58,8 @@ lint:
docker: sneak-mfer.$(ARCH).docker.tzst
sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES) vendor.tar
sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES)
bash -x bin/docker-prereqs.sh
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
docker save sneak/mfer | pv | zstdmt -19 > $@
du -sh $@
@ -66,8 +67,3 @@ sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES) vendor.tar
godoc:
open http://127.0.0.1:6060
godoc -http=:6060
vendor.tar: go.mod go.sum
go mod vendor
tar -c vendor > $@
rm -rf vendor

47
bin/docker-prereqs.sh Normal file
View File

@ -0,0 +1,47 @@
#!/bin/bash
set -euxo pipefail
GOI="golang:1.19.3-bullseye"
#CII="golangci/golangci-lint:v1.50.1"
if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then
mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
fi
DICD="$DOCKER_IMAGE_CACHE_DIR"
function buildImageCache() {
if [[ ! -e "$DICD/go.tzst" ]]; then
docker pull $GOI
docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \
mv $DICD/go.tzst.tmp $DICD/go.tzst
fi
#if [[ ! -e "$DICD/ci.tzst" ]]; then
# docker pull $CII
# docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \
# mv $DICD/ci.tzst.tmp $DICD/ci.tzst
#fi
}
function loadImageCache() {
# zstdmt -d --stdout $DICD/ci.tzst | pv | docker load
zstdmt -d --stdout $DICD/go.tzst | pv | docker load
#docker image ls $CII
docker image ls $GOI
}
function writeModuleCache() {
if [[ ! -e $DICD/modcache.tar ]]; then
cd $(go env GOMODCACHE)
tar -c . | pv > $DICD/modcache.tar.tmp && mv $DICD/modcache.tar.tmp $DICD/modcache.tar
cd -
fi
cp -av $DICD/modcache.tar ./builddeps/modcache.tar.tmp && mv ./builddeps/modcache.tar.tmp ./builddeps/modcache.tar
}
buildImageCache
loadImageCache
writeModuleCache

BIN
builddeps/protoc-21.10-linux-aarch64.zip Normal file
View File

Binary file not shown.

BIN
builddeps/protoc-21.10-linux-x86_64.zip Normal file
View File

Binary file not shown.

21
buildimage/Dockerfile Normal file
View File

@ -0,0 +1,21 @@
## build image:
FROM golang:1.19.3-bullseye AS builder
ENV DEBIAN_FRONTEND noninteractive
RUN apt update && apt install -y make bzip2 curl unzip
RUN mkdir -p /build
WORKDIR /build
# install newer protoc
RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \
unzip *.zip -d /usr/local && rm -v *.zip && protoc --version
RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
RUN go env
COPY ./go.mod .
COPY ./go.sum .
RUN --mount=type=cache,target=/go/pkg go mod download -x
RUN rm -rfv /var/cache/* /var/tmp/*

1
go.mod
View File

@ -25,6 +25,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect

10
go.sum
View File

@ -37,7 +37,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
@ -62,6 +61,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -189,9 +190,13 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
@ -216,6 +221,8 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
@ -530,6 +537,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

15
internal/bork/error.go
View File

@ -1,15 +0,0 @@
package bork
import (
"errors"
"fmt"
)
var (
ErrMissingMagic = errors.New("missing magic bytes in file")
ErrFileTruncated = errors.New("file/stream is truncated abnormally")
)
func Newf(format string, args ...interface{}) error {
return fmt.Errorf(format, args...)
}

11
internal/bork/error_test.go
View File

@ -1,11 +0,0 @@
package bork
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestBuild(t *testing.T) {
assert.NotNil(t, ErrMissingMagic)
}

32
internal/cli/misc.go Normal file
View File

@ -0,0 +1,32 @@
package cli
import "fmt"
// FIXME make this write to a bytes.Buffer with fprintf
func dumpByteSlice(b []byte) {
var a [16]byte
n := (len(b) + 15) &^ 15
for i := 0; i < n; i++ {
if i%16 == 0 {
fmt.Printf("%4d", i)
}
if i%8 == 0 {
fmt.Print(" ")
}
if i < len(b) {
fmt.Printf(" %02X", b[i])
} else {
fmt.Print(" ")
}
if i >= len(b) {
a[i%16] = ' '
} else if b[i] < 32 || b[i] > 126 {
a[i%16] = '.'
} else {
a[i%16] = b[i]
}
if i%16 == 15 {
fmt.Printf(" %s\n", string(a[:]))
}
}
}

21
internal/log/log.go
View File

@ -1,9 +1,6 @@
package log
import (
"fmt"
"runtime"
"github.com/apex/log"
acli "github.com/apex/log/handlers/cli"
"github.com/davecgh/go-spew/spew"
@ -28,25 +25,13 @@ func Init() {
log.SetLevel(log.InfoLevel)
}
func Debugf(format string, args ...interface{}) {
DebugReal(fmt.Sprintf(format, args...), 2)
}
func Debug(arg string) {
DebugReal(arg, 2)
}
func DebugReal(arg string, cs int) {
_, callerFile, callerLine, ok := runtime.Caller(cs)
if !ok {
return
}
tag := fmt.Sprintf("%s:%d: ", callerFile, callerLine)
log.Debug(tag + arg)
log.Debug(arg)
}
func Dump(args ...interface{}) {
DebugReal(spew.Sdump(args...), 2)
str := spew.Sdump(args...)
Debug(str)
}
func EnableDebugLogging() {

12
internal/log/log_test.go
View File

@ -1,12 +0,0 @@
package log
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestBuild(t *testing.T) {
Init()
assert.True(t, true)
}

89
mfer/deserialize.go
View File

@ -1,89 +0,0 @@
package mfer
import (
"bytes"
"compress/gzip"
"errors"
"io"
"git.eeqj.de/sneak/mfer/internal/bork"
"git.eeqj.de/sneak/mfer/internal/log"
"google.golang.org/protobuf/proto"
)
func (m *manifest) validateProtoOuter() error {
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
return errors.New("unknown version")
}
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
return errors.New("unknown compression type")
}
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
gzr, err := gzip.NewReader(bb)
if err != nil {
return err
}
dat, err := io.ReadAll(gzr)
defer gzr.Close()
if err != nil {
return err
}
isize := len(dat)
if int64(isize) != m.pbOuter.Size {
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
return bork.ErrFileTruncated
}
log.Debugf("inner data size is %d", isize)
log.Dump(dat)
log.Dump(m.pbOuter.Sha256)
return nil
}
func validateMagic(dat []byte) bool {
ml := len([]byte(MAGIC))
if len(dat) < ml {
return false
}
got := dat[0:ml]
expected := []byte(MAGIC)
return bytes.Equal(got, expected)
}
func NewFromProto(input io.Reader) (*manifest, error) {
m := New()
dat, err := io.ReadAll(input)
if err != nil {
return nil, err
}
if !validateMagic(dat) {
return nil, errors.New("invalid file format")
}
// remove magic bytes prefix:
ml := len([]byte(MAGIC))
bb := bytes.NewBuffer(dat[ml:])
dat = bb.Bytes()
log.Dump(dat)
// deserialize:
m.pbOuter = new(MFFileOuter)
err = proto.Unmarshal(dat, m.pbOuter)
if err != nil {
return nil, err
}
ve := m.validateProtoOuter()
if ve != nil {
return nil, ve
}
// FIXME TODO deserialize inner
return m, nil
}

42
mfer/example_test.go
View File

@ -1,42 +0,0 @@
package mfer
import (
"bytes"
"testing"
"git.eeqj.de/sneak/mfer/internal/log"
"github.com/stretchr/testify/assert"
)
func TestAPIExample(t *testing.T) {
// read from filesystem
m, err := NewFromFS(&ManifestScanOptions{
IgnoreDotfiles: true,
}, big)
assert.Nil(t, err)
assert.NotNil(t, m)
// scan for files
m.Scan()
// serialize
var buf bytes.Buffer
m.WriteTo(&buf)
// show serialized
log.Dump(buf.Bytes())
// do it again
var buf2 bytes.Buffer
m.WriteTo(&buf2)
// should be same!
assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
// deserialize
m2, err := NewFromProto(&buf)
assert.Nil(t, err)
assert.NotNil(t, m2)
log.Dump(m2)
}

4
mfer/manifest.go
View File

@ -65,7 +65,8 @@ func (m *manifest) addInputPath(inputPath string) error {
}
// FIXME check to make sure inputPath/abs exists maybe
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs))
return m.addInputFS(afs)
m.addInputFS(afs)
return nil
}
func (m *manifest) addInputFS(f afero.Fs) error {
@ -158,6 +159,7 @@ func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
func (m *manifest) Scan() error {
// FIXME scan and whatever function does the hashing should take ctx
log.Debug("manifest scanning")
for idx, sfs := range m.sourceFS {
if sfs == nil {
return errors.New("invalid source fs")

19
mfer/mf.proto
View File

@ -9,26 +9,19 @@ message Timestamp {
message MFFileOuter {
enum Version {
VERSION_NONE = 0;
VERSION_ONE = 1; // only one for now
NONE = 0;
ONE = 1; // only one for now
}
// required mffile root attributes 1xx
Version version = 101;
enum CompressionType {
COMPRESSION_NONE = 0;
COMPRESSION_GZIP = 1;
}
CompressionType compressionType = 102;
bytes innerMessage = 102;
// these are used solely to detect corruption/truncation
// and not for cryptographic integrity.
int64 size = 103;
bytes sha256 = 104;
bytes deleteme = 105;
bytes innerMessage = 199;
// 2xx for optional manifest root attributes
// think we might use gosignify instead of gpg:
// github.com/frankbraun/gosignify
@ -64,8 +57,8 @@ message MFFileChecksum {
message MFFile {
enum Version {
VERSION_NONE = 0;
VERSION_ONE = 1; // only one for now
NONE = 0;
ONE = 1; // only one for now
}
Version version = 100;

27
mfer/mfer_test.go
View File

@ -2,10 +2,10 @@ package mfer
import (
"bytes"
"fmt"
"testing"
"git.eeqj.de/sneak/mfer/internal/log"
"github.com/davecgh/go-spew/spew"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
)
@ -16,25 +16,19 @@ var (
)
var (
af *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
big *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
mf afero.Fs = afero.NewMemMapFs()
af *afero.Afero = &afero.Afero{Fs: mf}
)
func init() {
log.EnableDebugLogging()
// create test files and directories
af.MkdirAll("/a/b/c", 0o755)
af.MkdirAll("/.hidden", 0o755)
af.WriteFile("/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
af.WriteFile("/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
af.WriteFile("/.hidden/hello.txt", []byte("hello world\n"), 0o755)
af.WriteFile("/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
big.MkdirAll("/home/user/Library", 0o755)
for i := range [25]int{} {
big.WriteFile(fmt.Sprintf("/home/user/Library/hello%d.txt", i), []byte("hello world\n"), 0o755)
}
afero.WriteFile(af, "/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
afero.WriteFile(af, "/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
afero.WriteFile(af, "/.hidden/hello.txt", []byte("hello world\n"), 0o755)
afero.WriteFile(af, "/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
log.EnableDebugLogging()
}
func TestPathHiddenFunc(t *testing.T) {
@ -51,7 +45,6 @@ func TestManifestGenerationOne(t *testing.T) {
}, af)
assert.Nil(t, err)
assert.NotNil(t, m)
m.Scan()
assert.Equal(t, int64(2), m.GetFileCount())
assert.Equal(t, int64(30), m.GetTotalFileSize())
}
@ -62,7 +55,7 @@ func TestManifestGenerationTwo(t *testing.T) {
}, af)
assert.Nil(t, err)
assert.NotNil(t, m)
m.Scan()
spew.Dump(m)
assert.Equal(t, int64(4), m.GetFileCount())
assert.Equal(t, int64(54), m.GetTotalFileSize())
err = m.generate()
@ -70,5 +63,5 @@ func TestManifestGenerationTwo(t *testing.T) {
var buf bytes.Buffer
err = m.WriteTo(&buf)
assert.Nil(t, err)
log.Dump(buf.Bytes())
spew.Dump(buf)
}

42
mfer/serialize.go
View File

@ -2,19 +2,15 @@ package mfer
import (
"bytes"
"compress/gzip"
"crypto/sha256"
"errors"
"time"
"git.eeqj.de/sneak/mfer/internal/log"
"google.golang.org/protobuf/proto"
)
//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
// rot13("MANIFEST")
const MAGIC string = "ZNAVSRFG"
func newTimestampFromTime(t time.Time) *Timestamp {
out := &Timestamp{
Seconds: t.Unix(),
@ -24,6 +20,10 @@ func newTimestampFromTime(t time.Time) *Timestamp {
}
func (m *manifest) generate() error {
log.Debug("generate()")
const MAGIC string = "ZNAVSRFG"
if m.pbInner == nil {
e := m.generateInner()
if e != nil {
@ -36,7 +36,7 @@ func (m *manifest) generate() error {
return e
}
}
dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
dat, err := proto.Marshal(m.pbOuter)
if err != nil {
return err
}
@ -49,46 +49,29 @@ func (m *manifest) generate() error {
}
func (m *manifest) generateOuter() error {
log.Debug("generateOuter()")
if m.pbInner == nil {
return errors.New("internal error")
}
innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
innerData, err := proto.Marshal(m.pbInner)
if err != nil {
return err
}
h := sha256.New()
h.Write(innerData)
idc := new(bytes.Buffer)
gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
if err != nil {
return err
}
_, err = gzw.Write(innerData)
if err != nil {
return err
}
gzw.Close()
o := &MFFileOuter{
InnerMessage: idc.Bytes(),
Size: int64(len(innerData)),
Sha256: h.Sum(nil),
Version: MFFileOuter_VERSION_ONE,
CompressionType: MFFileOuter_COMPRESSION_GZIP,
InnerMessage: innerData,
}
m.pbOuter = o
return nil
}
func (m *manifest) generateInner() error {
log.Debug("generateInner()")
m.pbInner = &MFFile{
Version: MFFile_VERSION_ONE,
Version: MFFile_ONE,
CreatedAt: newTimestampFromTime(time.Now()),
Files: []*MFFilePath{},
}
for _, f := range m.files {
nf := &MFFilePath{
Path: f.path,
@ -96,5 +79,6 @@ func (m *manifest) generateInner() error {
}
m.pbInner.Files = append(m.pbInner.Files, nf)
}
log.Dump(m.pbInner)
return nil
}

BIN
vendor.tar
View File

Binary file not shown.