getting warmer

This commit is contained in:
Jeffrey Paul 2022-12-06 20:42:26 +04:00
parent b0c16462c4
commit 86d724ee35
6 changed files with 146 additions and 19 deletions

13
internal/bork/error.go Normal file
View File

@ -0,0 +1,13 @@
package bork
import (
"errors"
"fmt"
)
var ErrMissingMagic = errors.New("missing magic bytes in file")
var ErrFileTruncated = errors.New("file/stream is truncated abnormally")
func Newf(format string, args ...interface{}) error {
return errors.New(fmt.Sprintf(format, args...))
}

View File

@ -1,6 +1,9 @@
package log package log
import ( import (
"fmt"
"runtime"
"github.com/apex/log" "github.com/apex/log"
acli "github.com/apex/log/handlers/cli" acli "github.com/apex/log/handlers/cli"
"github.com/davecgh/go-spew/spew" "github.com/davecgh/go-spew/spew"
@ -25,13 +28,25 @@ func Init() {
log.SetLevel(log.InfoLevel) log.SetLevel(log.InfoLevel)
} }
func Debugf(format string, args ...interface{}) {
DebugReal(fmt.Sprintf(format, args...), 2)
}
func Debug(arg string) { func Debug(arg string) {
log.Debug(arg) DebugReal(arg, 2)
}
func DebugReal(arg string, cs int) {
_, callerFile, callerLine, ok := runtime.Caller(cs)
if !ok {
return
}
tag := fmt.Sprintf("%s:%d: ", callerFile, callerLine)
log.Debug(tag + arg)
} }
func Dump(args ...interface{}) { func Dump(args ...interface{}) {
str := spew.Sdump(args...) DebugReal(spew.Sdump(args...), 2)
Debug(str)
} }
func EnableDebugLogging() { func EnableDebugLogging() {

View File

@ -1,12 +1,81 @@
package mfer package mfer
import "google.golang.org/protobuf/proto" import (
"bytes"
"compress/gzip"
"errors"
"io"
func NewFromProto(input []byte) (*manifest, error) { "git.eeqj.de/sneak/mfer/internal/bork"
"git.eeqj.de/sneak/mfer/internal/log"
"google.golang.org/protobuf/proto"
)
func (m *manifest) validateProto() error {
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
return errors.New("unknown version")
}
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
return errors.New("unknown compression type")
}
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
gzr, err := gzip.NewReader(bb)
if err != nil {
return err
}
dat, err := io.ReadAll(gzr)
defer gzr.Close()
if err != nil {
return err
}
isize := len(dat)
if int64(isize) != m.pbOuter.Size {
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
return bork.ErrFileTruncated
}
log.Dump(dat)
log.Dump(m.pbOuter.Sha256)
return nil
}
func NewFromProto(input io.Reader) (*manifest, error) {
m := New() m := New()
err := proto.Unmarshal(input, m.pbOuter) dat, err := io.ReadAll(input)
if err != nil { if err != nil {
return nil, err return nil, err
} }
ml := len([]byte(MAGIC))
bb := bytes.NewBuffer(dat)
got := dat[0:ml]
log.Dump("got:")
log.Dump(got)
expected := []byte(MAGIC)
log.Dump("expected:")
log.Dump(expected)
if !bytes.Equal(got, expected) {
return nil, errors.New("invalid file format")
}
bb = bytes.NewBuffer(dat[ml:])
dat = bb.Bytes()
log.Dump(dat)
m.pbOuter = new(MFFileOuter)
err = proto.Unmarshal(dat, m.pbOuter)
if err != nil {
return nil, err
}
ve := m.validateProto()
if ve != nil {
return nil, ve
}
return m, nil return m, nil
} }

View File

@ -4,17 +4,40 @@ import (
"bytes" "bytes"
"testing" "testing"
"github.com/davecgh/go-spew/spew" "git.eeqj.de/sneak/mfer/internal/log"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func TestAPIExample(t *testing.T) { func TestAPIExample(t *testing.T) {
// read from filesystem
m, err := NewFromFS(&ManifestScanOptions{ m, err := NewFromFS(&ManifestScanOptions{
IgnoreDotfiles: true, IgnoreDotfiles: true,
}, af) }, big)
assert.NotNil(t, err) assert.Nil(t, err)
assert.NotNil(t, m)
// scan for files
m.Scan() m.Scan()
// serialize
var buf bytes.Buffer var buf bytes.Buffer
m.WriteTo(&buf) m.WriteTo(&buf)
spew.Dump(buf.Bytes())
// show serialized
log.Dump(buf.Bytes())
// do it again
var buf2 bytes.Buffer
m.WriteTo(&buf2)
// should be same!
assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
// deserialize
m2, err := NewFromProto(&buf)
assert.Nil(t, err)
assert.NotNil(t, m2)
log.Dump(m2)
} }

View File

@ -2,6 +2,7 @@ package mfer
import ( import (
"bytes" "bytes"
"fmt"
"testing" "testing"
"git.eeqj.de/sneak/mfer/internal/log" "git.eeqj.de/sneak/mfer/internal/log"
@ -15,19 +16,25 @@ var (
) )
var ( var (
mf afero.Fs = afero.NewMemMapFs() af *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
af *afero.Afero = &afero.Afero{Fs: mf} big *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
) )
func init() { func init() {
log.EnableDebugLogging()
// create test files and directories // create test files and directories
af.MkdirAll("/a/b/c", 0o755) af.MkdirAll("/a/b/c", 0o755)
af.MkdirAll("/.hidden", 0o755) af.MkdirAll("/.hidden", 0o755)
afero.WriteFile(af, "/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755) af.WriteFile("/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
afero.WriteFile(af, "/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755) af.WriteFile("/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
afero.WriteFile(af, "/.hidden/hello.txt", []byte("hello world\n"), 0o755) af.WriteFile("/.hidden/hello.txt", []byte("hello world\n"), 0o755)
afero.WriteFile(af, "/.hidden/hello2.txt", []byte("hello world\n"), 0o755) af.WriteFile("/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
log.EnableDebugLogging()
big.MkdirAll("/home/user/Library", 0o755)
for i, _ := range [25]int{} {
big.WriteFile(fmt.Sprintf("/home/user/Library/hello%d.txt", i), []byte("hello world\n"), 0o755)
}
} }
func TestPathHiddenFunc(t *testing.T) { func TestPathHiddenFunc(t *testing.T) {

View File

@ -13,6 +13,8 @@ import (
//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto //go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
const MAGIC string = "ZNAVSRFG"
func newTimestampFromTime(t time.Time) *Timestamp { func newTimestampFromTime(t time.Time) *Timestamp {
out := &Timestamp{ out := &Timestamp{
Seconds: t.Unix(), Seconds: t.Unix(),
@ -24,8 +26,6 @@ func newTimestampFromTime(t time.Time) *Timestamp {
func (m *manifest) generate() error { func (m *manifest) generate() error {
log.Debug("generate()") log.Debug("generate()")
const MAGIC string = "ZNAVSRFG"
if m.pbInner == nil { if m.pbInner == nil {
e := m.generateInner() e := m.generateInner()
if e != nil { if e != nil {