sneak
/
mfer
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Compare commits

base: sneak:next
Branches Tags
sneak:main
sneak:next
..
compare: sneak:main
Branches Tags
sneak:next
sneak:main

1 Commits
next ... main

Author SHA1 Message Date
Jeffrey Paul 7df558d8d0 next (#5)
continuous-integration/drone/push Build is passing Details 
Co-authored-by: sneak <sneak@sneak.berlin>
Reviewed-on: #5
 2022-12-09 00:02:33 +00:00
9 changed files with 35 additions and 131 deletions
Show Stats Expand all files Collapse all files

3
TODO.md
View File

@ -1,3 +0,0 @@
# TODO
* add github.com/schollz/progressbar

6
internal/bork/error.go
View File

@ -2,10 +2,14 @@ package bork
import (
"errors"
"fmt"
)
var (
ErrMissingMagic = errors.New("missing magic bytes in file")
ErrFileTruncated = errors.New("file/stream is truncated abnormally")
ErrFileIntegrity = errors.New("file/stream checksum failure")
)
func Newf(format string, args ...interface{}) error {
return fmt.Errorf(format, args...)
}

21
internal/log/log.go
View File

@ -23,20 +23,11 @@ func DisableStyling() {
pterm.Fatal.Prefix.Text = ""
}
var TestingTraces bool = false
func Init() {
log.SetHandler(acli.Default)
log.SetLevel(log.InfoLevel)
}
func Tracef(format string, args ...interface{}) {
if !TestingTraces {
return
}
DebugReal(fmt.Sprintf(format, args...), 2)
}
func Debugf(format string, args ...interface{}) {
DebugReal(fmt.Sprintf(format, args...), 2)
}
@ -54,22 +45,10 @@ func DebugReal(arg string, cs int) {
log.Debug(tag + arg)
}
func TraceDump(args ...interface{}) {
if !TestingTraces {
return
}
DebugReal(spew.Sdump(args...), 2)
}
func Dump(args ...interface{}) {
DebugReal(spew.Sdump(args...), 2)
}
func EnableTestingLogging() {
TestingTraces = true
EnableDebugLogging()
}
func EnableDebugLogging() {
SetLevel(log.DebugLevel)
}

72
mfer/deserialize.go
View File

@ -3,7 +3,6 @@ package mfer
import (
"bytes"
"compress/gzip"
"crypto/sha256"
"errors"
"io"
@ -12,35 +11,12 @@ import (
"google.golang.org/protobuf/proto"
)
func (m *manifest) validateProtoInner() error {
i := m.pbInner
if i.Version != MFFile_VERSION_ONE {
return errors.New("unknown version") // FIXME move to bork
}
if len(i.Files) == 0 {
return errors.New("manifest without files") // FIXME move to bork
}
for _, mfp := range m.pbInner.Files {
// there is no way we should be doing validateProtoInner()
// outside of a load into a blank/empty/new *manifest
if m.files != nil {
return errors.New("shouldn't happen, internal error")
}
m.files = make([]*manifestFile, 0)
// we can skip error handling here thanks to the magic of protobuf
m.addFileLoadTime(mfp)
}
return nil
}
func (m *manifest) validateProtoOuter() error {
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
return errors.New("unknown version") // FIXME move to bork
return errors.New("unknown version")
}
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
return errors.New("unknown compression type") // FIXME move to bork
return errors.New("unknown compression type")
}
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
@ -59,26 +35,12 @@ func (m *manifest) validateProtoOuter() error {
isize := len(dat)
if int64(isize) != m.pbOuter.Size {
log.Tracef("truncated data, got %d expected %d", isize, m.pbOuter.Size)
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
return bork.ErrFileTruncated
}
log.Tracef("inner data size is %d", isize)
log.TraceDump(dat)
// FIXME validate Sha256
log.TraceDump(m.pbOuter.Sha256)
h := sha256.New()
h.Write(dat)
shaGot := h.Sum(nil)
log.TraceDump("got: ", shaGot)
log.TraceDump("expected: ", m.pbOuter.Sha256)
if !bytes.Equal(shaGot, m.pbOuter.Sha256) {
m.pbOuter.InnerMessage = nil // don't try to mess with it
return bork.ErrFileIntegrity
}
log.Debugf("inner data size is %d", isize)
log.Dump(dat)
log.Dump(m.pbOuter.Sha256)
return nil
}
@ -92,7 +54,7 @@ func validateMagic(dat []byte) bool {
return bytes.Equal(got, expected)
}
func NewFromReader(input io.Reader) (*manifest, error) {
func NewFromProto(input io.Reader) (*manifest, error) {
m := New()
dat, err := io.ReadAll(input)
if err != nil {
@ -107,7 +69,7 @@ func NewFromReader(input io.Reader) (*manifest, error) {
bb := bytes.NewBuffer(dat[ml:])
dat = bb.Bytes()
log.TraceDump(dat)
log.Dump(dat)
// deserialize:
m.pbOuter = new(MFFileOuter)
@ -122,22 +84,6 @@ func NewFromReader(input io.Reader) (*manifest, error) {
return nil, ve
}
m.pbInner = new(MFFile)
err = proto.Unmarshal(m.pbOuter.InnerMessage, m.pbInner)
if err != nil {
return nil, err
}
log.TraceDump(m.pbInner)
ve = m.validateProtoInner()
if ve != nil {
return nil, ve
}
m.rescanInternal()
log.TraceDump(m)
// FIXME TODO deserialize inner
return m, nil
}

10
mfer/example_test.go
View File

@ -8,10 +8,6 @@ import (
"github.com/stretchr/testify/assert"
)
func init() {
log.EnableTestingLogging()
}
func TestAPIExample(t *testing.T) {
// read from filesystem
m, err := NewFromFS(&ManifestScanOptions{
@ -28,7 +24,7 @@ func TestAPIExample(t *testing.T) {
m.WriteTo(&buf)
// show serialized
log.TraceDump(buf.Bytes())
log.Dump(buf.Bytes())
// do it again
var buf2 bytes.Buffer
@ -38,9 +34,9 @@ func TestAPIExample(t *testing.T) {
assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
// deserialize
m2, err := NewFromReader(&buf)
m2, err := NewFromProto(&buf)
assert.Nil(t, err)
assert.NotNil(t, m2)
log.TraceDump(m2)
log.Dump(m2)
}

26
mfer/manifest.go
View File

@ -15,10 +15,8 @@ import (
)
type manifestFile struct {
path string
info fs.FileInfo
size int64
protoFile *MFFilePath
path string
info fs.FileInfo
}
func (m *manifestFile) String() string {
@ -85,7 +83,7 @@ func New() *manifest {
}
func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) {
log.TraceDump(inputPaths)
log.Dump(inputPaths)
m := New()
m.scanOptions = options
for _, p := range inputPaths {
@ -111,10 +109,6 @@ func (m *manifest) GetFileCount() int64 {
return int64(len(m.files))
}
func (m *manifest) rescanInternal() {
panic("unimplemented")
}
func (m *manifest) GetTotalFileSize() int64 {
return m.totalFileSize
}
@ -136,17 +130,7 @@ func pathIsHidden(p string) bool {
}
}
func (m *manifest) addFileLoadTime(in *MFFilePath) {
nf := &manifestFile{
path: in.Path,
size: in.Size,
protoFile: in, // FIXME get rid of this eventually
}
m.files = append(m.files, nf)
m.totalFileSize = m.totalFileSize + in.Size
}
func (m *manifest) addFileScanTime(p string, fi fs.FileInfo, sfsIndex int) error {
func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) {
return nil
}
@ -179,7 +163,7 @@ func (m *manifest) Scan() error {
return errors.New("invalid source fs")
}
e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error {
return m.addFileScanTime(p, info, idx)
return m.addFile(p, info, idx)
})
if e != nil {
return e

18
mfer/mf.proto
View File

@ -9,7 +9,7 @@ message Timestamp {
message MFFileOuter {
enum Version {
VERSION_NONE = 0; // must have a zero enum by law
VERSION_NONE = 0;
VERSION_ONE = 1; // only one for now
}
@ -17,7 +17,7 @@ message MFFileOuter {
Version version = 101;
enum CompressionType {
COMPRESSION_NONE = 0; // must have a zero enum by law
COMPRESSION_NONE = 0;
COMPRESSION_GZIP = 1;
}
@ -33,10 +33,12 @@ message MFFileOuter {
// think we might use gosignify instead of gpg:
// github.com/frankbraun/gosignify
//detached signatures, ascii or binary
repeated bytes signatures = 201;
//full GPG signing public keys, ascii or binary
repeated bytes signingPubKeys = 203;
//detached signature, ascii or binary
optional bytes signature = 201;
//full GPG key id
optional bytes signer = 202;
//full GPG signing public key, ascii or binary
optional bytes signingPubKey = 203;
}
message MFFilePath {
@ -56,13 +58,13 @@ message MFFilePath {
message MFFileChecksum {
// 1.0 golang implementation must write a multihash here
// it's ok to only ever use/verify sha256 multihash i think?
// it's ok to only ever use/verify sha256 multihash
bytes multiHash = 1;
}
message MFFile {
enum Version {
VERSION_NONE = 0; // must have a zero enum by law
VERSION_NONE = 0;
VERSION_ONE = 1; // only one for now
}
Version version = 100;

4
mfer/mfer_test.go
View File

@ -21,7 +21,7 @@ var (
)
func init() {
log.EnableTestingLogging()
log.EnableDebugLogging()
// create test files and directories
af.MkdirAll("/a/b/c", 0o755)
@ -70,5 +70,5 @@ func TestManifestGenerationTwo(t *testing.T) {
var buf bytes.Buffer
err = m.WriteTo(&buf)
assert.Nil(t, err)
log.TraceDump(buf.Bytes())
log.Dump(buf.Bytes())
}

6
mfer/serialize.go
View File

@ -7,7 +7,6 @@ import (
"errors"
"time"
"git.eeqj.de/sneak/mfer/internal/log"
"google.golang.org/protobuf/proto"
)
@ -60,7 +59,6 @@ func (m *manifest) generateOuter() error {
h := sha256.New()
h.Write(innerData)
sha256 := h.Sum(nil)
idc := new(bytes.Buffer)
gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
@ -74,12 +72,10 @@ func (m *manifest) generateOuter() error {
gzw.Close()
log.Tracef("calculated sha256 (uncompressed): %x\n", sha256)
o := &MFFileOuter{
InnerMessage: idc.Bytes(),
Size: int64(len(innerData)),
Sha256: sha256,
Sha256: h.Sum(nil),
Version: MFFileOuter_VERSION_ONE,
CompressionType: MFFileOuter_COMPRESSION_GZIP,
}