diff --git a/internal/bork/error.go b/internal/bork/error.go index 6ead94d..dabbf8b 100644 --- a/internal/bork/error.go +++ b/internal/bork/error.go @@ -2,14 +2,10 @@ package bork import ( "errors" - "fmt" ) var ( ErrMissingMagic = errors.New("missing magic bytes in file") ErrFileTruncated = errors.New("file/stream is truncated abnormally") + ErrFileIntegrity = errors.New("file/stream checksum failure") ) - -func Newf(format string, args ...interface{}) error { - return fmt.Errorf(format, args...) -} diff --git a/internal/log/log.go b/internal/log/log.go index b52409b..fe568e1 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -23,11 +23,20 @@ func DisableStyling() { pterm.Fatal.Prefix.Text = "" } +var TestingTraces bool = false + func Init() { log.SetHandler(acli.Default) log.SetLevel(log.InfoLevel) } +func Tracef(format string, args ...interface{}) { + if !TestingTraces { + return + } + DebugReal(fmt.Sprintf(format, args...), 2) +} + func Debugf(format string, args ...interface{}) { DebugReal(fmt.Sprintf(format, args...), 2) } @@ -45,10 +54,22 @@ func DebugReal(arg string, cs int) { log.Debug(tag + arg) } +func TraceDump(args ...interface{}) { + if !TestingTraces { + return + } + DebugReal(spew.Sdump(args...), 2) +} + func Dump(args ...interface{}) { DebugReal(spew.Sdump(args...), 2) } +func EnableTestingLogging() { + TestingTraces = true + EnableDebugLogging() +} + func EnableDebugLogging() { SetLevel(log.DebugLevel) } diff --git a/mfer/deserialize.go b/mfer/deserialize.go index 85ff318..1cc4f46 100644 --- a/mfer/deserialize.go +++ b/mfer/deserialize.go @@ -3,6 +3,7 @@ package mfer import ( "bytes" "compress/gzip" + "crypto/sha256" "errors" "io" @@ -11,12 +12,35 @@ import ( "google.golang.org/protobuf/proto" ) +func (m *manifest) validateProtoInner() error { + i := m.pbInner + if i.Version != MFFile_VERSION_ONE { + return errors.New("unknown version") // FIXME move to bork + } + + if len(i.Files) == 0 { + return errors.New("manifest without files") // FIXME move to bork + } + + for _, mfp := range m.pbInner.Files { + // there is no way we should be doing validateProtoInner() + // outside of a load into a blank/empty/new *manifest + if m.files != nil { + return errors.New("shouldn't happen, internal error") + } + m.files = make([]*manifestFile, 0) + // we can skip error handling here thanks to the magic of protobuf + m.addFileLoadTime(mfp) + } + return nil +} + func (m *manifest) validateProtoOuter() error { if m.pbOuter.Version != MFFileOuter_VERSION_ONE { - return errors.New("unknown version") + return errors.New("unknown version") // FIXME move to bork } if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP { - return errors.New("unknown compression type") + return errors.New("unknown compression type") // FIXME move to bork } bb := bytes.NewBuffer(m.pbOuter.InnerMessage) @@ -35,12 +59,26 @@ func (m *manifest) validateProtoOuter() error { isize := len(dat) if int64(isize) != m.pbOuter.Size { - log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size) + log.Tracef("truncated data, got %d expected %d", isize, m.pbOuter.Size) return bork.ErrFileTruncated } - log.Debugf("inner data size is %d", isize) - log.Dump(dat) - log.Dump(m.pbOuter.Sha256) + log.Tracef("inner data size is %d", isize) + log.TraceDump(dat) + + // FIXME validate Sha256 + log.TraceDump(m.pbOuter.Sha256) + + h := sha256.New() + h.Write(dat) + shaGot := h.Sum(nil) + + log.TraceDump("got: ", shaGot) + log.TraceDump("expected: ", m.pbOuter.Sha256) + + if !bytes.Equal(shaGot, m.pbOuter.Sha256) { + m.pbOuter.InnerMessage = nil // don't try to mess with it + return bork.ErrFileIntegrity + } return nil } @@ -54,7 +92,7 @@ func validateMagic(dat []byte) bool { return bytes.Equal(got, expected) } -func NewFromProto(input io.Reader) (*manifest, error) { +func NewFromReader(input io.Reader) (*manifest, error) { m := New() dat, err := io.ReadAll(input) if err != nil { @@ -69,7 +107,7 @@ func NewFromProto(input io.Reader) (*manifest, error) { bb := bytes.NewBuffer(dat[ml:]) dat = bb.Bytes() - log.Dump(dat) + log.TraceDump(dat) // deserialize: m.pbOuter = new(MFFileOuter) @@ -84,6 +122,22 @@ func NewFromProto(input io.Reader) (*manifest, error) { return nil, ve } - // FIXME TODO deserialize inner + m.pbInner = new(MFFile) + err = proto.Unmarshal(m.pbOuter.InnerMessage, m.pbInner) + if err != nil { + return nil, err + } + + log.TraceDump(m.pbInner) + + ve = m.validateProtoInner() + if ve != nil { + return nil, ve + } + + m.rescanInternal() + + log.TraceDump(m) + return m, nil } diff --git a/mfer/example_test.go b/mfer/example_test.go index af4164b..c2c25d8 100644 --- a/mfer/example_test.go +++ b/mfer/example_test.go @@ -8,6 +8,10 @@ import ( "github.com/stretchr/testify/assert" ) +func init() { + log.EnableTestingLogging() +} + func TestAPIExample(t *testing.T) { // read from filesystem m, err := NewFromFS(&ManifestScanOptions{ @@ -24,7 +28,7 @@ func TestAPIExample(t *testing.T) { m.WriteTo(&buf) // show serialized - log.Dump(buf.Bytes()) + log.TraceDump(buf.Bytes()) // do it again var buf2 bytes.Buffer @@ -34,9 +38,9 @@ func TestAPIExample(t *testing.T) { assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes())) // deserialize - m2, err := NewFromProto(&buf) + m2, err := NewFromReader(&buf) assert.Nil(t, err) assert.NotNil(t, m2) - log.Dump(m2) + log.TraceDump(m2) } diff --git a/mfer/manifest.go b/mfer/manifest.go index d85e5c5..4ea8f06 100644 --- a/mfer/manifest.go +++ b/mfer/manifest.go @@ -15,8 +15,10 @@ import ( ) type manifestFile struct { - path string - info fs.FileInfo + path string + info fs.FileInfo + size int64 + protoFile *MFFilePath } func (m *manifestFile) String() string { @@ -83,7 +85,7 @@ func New() *manifest { } func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) { - log.Dump(inputPaths) + log.TraceDump(inputPaths) m := New() m.scanOptions = options for _, p := range inputPaths { @@ -109,6 +111,10 @@ func (m *manifest) GetFileCount() int64 { return int64(len(m.files)) } +func (m *manifest) rescanInternal() { + panic("unimplemented") +} + func (m *manifest) GetTotalFileSize() int64 { return m.totalFileSize } @@ -130,7 +136,17 @@ func pathIsHidden(p string) bool { } } -func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error { +func (m *manifest) addFileLoadTime(in *MFFilePath) { + nf := &manifestFile{ + path: in.Path, + size: in.Size, + protoFile: in, // FIXME get rid of this eventually + } + m.files = append(m.files, nf) + m.totalFileSize = m.totalFileSize + in.Size +} + +func (m *manifest) addFileScanTime(p string, fi fs.FileInfo, sfsIndex int) error { if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) { return nil } @@ -163,7 +179,7 @@ func (m *manifest) Scan() error { return errors.New("invalid source fs") } e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error { - return m.addFile(p, info, idx) + return m.addFileScanTime(p, info, idx) }) if e != nil { return e diff --git a/mfer/mf.proto b/mfer/mf.proto index ebac757..a30eee1 100644 --- a/mfer/mf.proto +++ b/mfer/mf.proto @@ -9,7 +9,7 @@ message Timestamp { message MFFileOuter { enum Version { - VERSION_NONE = 0; + VERSION_NONE = 0; // must have a zero enum by law VERSION_ONE = 1; // only one for now } @@ -17,7 +17,7 @@ message MFFileOuter { Version version = 101; enum CompressionType { - COMPRESSION_NONE = 0; + COMPRESSION_NONE = 0; // must have a zero enum by law COMPRESSION_GZIP = 1; } @@ -33,12 +33,10 @@ message MFFileOuter { // think we might use gosignify instead of gpg: // github.com/frankbraun/gosignify - //detached signature, ascii or binary - optional bytes signature = 201; - //full GPG key id - optional bytes signer = 202; - //full GPG signing public key, ascii or binary - optional bytes signingPubKey = 203; + //detached signatures, ascii or binary + repeated bytes signatures = 201; + //full GPG signing public keys, ascii or binary + repeated bytes signingPubKeys = 203; } message MFFilePath { @@ -58,13 +56,13 @@ message MFFilePath { message MFFileChecksum { // 1.0 golang implementation must write a multihash here - // it's ok to only ever use/verify sha256 multihash + // it's ok to only ever use/verify sha256 multihash i think? bytes multiHash = 1; } message MFFile { enum Version { - VERSION_NONE = 0; + VERSION_NONE = 0; // must have a zero enum by law VERSION_ONE = 1; // only one for now } Version version = 100; diff --git a/mfer/mfer_test.go b/mfer/mfer_test.go index 18858a6..2b1f380 100644 --- a/mfer/mfer_test.go +++ b/mfer/mfer_test.go @@ -21,7 +21,7 @@ var ( ) func init() { - log.EnableDebugLogging() + log.EnableTestingLogging() // create test files and directories af.MkdirAll("/a/b/c", 0o755) @@ -70,5 +70,5 @@ func TestManifestGenerationTwo(t *testing.T) { var buf bytes.Buffer err = m.WriteTo(&buf) assert.Nil(t, err) - log.Dump(buf.Bytes()) + log.TraceDump(buf.Bytes()) } diff --git a/mfer/serialize.go b/mfer/serialize.go index 00e8a5e..187714e 100644 --- a/mfer/serialize.go +++ b/mfer/serialize.go @@ -7,6 +7,7 @@ import ( "errors" "time" + "git.eeqj.de/sneak/mfer/internal/log" "google.golang.org/protobuf/proto" ) @@ -59,6 +60,7 @@ func (m *manifest) generateOuter() error { h := sha256.New() h.Write(innerData) + sha256 := h.Sum(nil) idc := new(bytes.Buffer) gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression) @@ -72,10 +74,12 @@ func (m *manifest) generateOuter() error { gzw.Close() + log.Tracef("calculated sha256 (uncompressed): %x\n", sha256) + o := &MFFileOuter{ InnerMessage: idc.Bytes(), Size: int64(len(innerData)), - Sha256: h.Sum(nil), + Sha256: sha256, Version: MFFileOuter_VERSION_ONE, CompressionType: MFFileOuter_COMPRESSION_GZIP, }