next (#5)

Co-authored-by: sneak <sneak@sneak.berlin>
Reviewed-on: #5

mfer/deserialize.go

@@ -0,0 +1,89 @@
+package mfer
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"io"
+
+	"git.eeqj.de/sneak/mfer/internal/bork"
+	"git.eeqj.de/sneak/mfer/internal/log"
+	"google.golang.org/protobuf/proto"
+)
+
+func (m *manifest) validateProtoOuter() error {
+	if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
+		return errors.New("unknown version")
+	}
+	if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
+		return errors.New("unknown compression type")
+	}
+
+	bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
+
+	gzr, err := gzip.NewReader(bb)
+	if err != nil {
+		return err
+	}
+
+	dat, err := io.ReadAll(gzr)
+	defer gzr.Close()
+
+	if err != nil {
+		return err
+	}
+
+	isize := len(dat)
+	if int64(isize) != m.pbOuter.Size {
+		log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
+		return bork.ErrFileTruncated
+	}
+	log.Debugf("inner data size is %d", isize)
+	log.Dump(dat)
+	log.Dump(m.pbOuter.Sha256)
+	return nil
+}
+
+func validateMagic(dat []byte) bool {
+	ml := len([]byte(MAGIC))
+	if len(dat) < ml {
+		return false
+	}
+	got := dat[0:ml]
+	expected := []byte(MAGIC)
+	return bytes.Equal(got, expected)
+}
+
+func NewFromProto(input io.Reader) (*manifest, error) {
+	m := New()
+	dat, err := io.ReadAll(input)
+	if err != nil {
+		return nil, err
+	}
+	if !validateMagic(dat) {
+		return nil, errors.New("invalid file format")
+	}
+
+	// remove magic bytes prefix:
+	ml := len([]byte(MAGIC))
+	bb := bytes.NewBuffer(dat[ml:])
+	dat = bb.Bytes()
+
+	log.Dump(dat)
+
+	// deserialize:
+	m.pbOuter = new(MFFileOuter)
+	err = proto.Unmarshal(dat, m.pbOuter)
+
+	if err != nil {
+		return nil, err
+	}
+
+	ve := m.validateProtoOuter()
+	if ve != nil {
+		return nil, ve
+	}
+
+	// FIXME TODO deserialize inner
+	return m, nil
+}

mfer/example_test.go

@@ -0,0 +1,42 @@
+package mfer
+
+import (
+	"bytes"
+	"testing"
+
+	"git.eeqj.de/sneak/mfer/internal/log"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAPIExample(t *testing.T) {
+	// read from filesystem
+	m, err := NewFromFS(&ManifestScanOptions{
+		IgnoreDotfiles: true,
+	}, big)
+	assert.Nil(t, err)
+	assert.NotNil(t, m)
+
+	// scan for files
+	m.Scan()
+
+	// serialize
+	var buf bytes.Buffer
+	m.WriteTo(&buf)
+
+	// show serialized
+	log.Dump(buf.Bytes())
+
+	// do it again
+	var buf2 bytes.Buffer
+	m.WriteTo(&buf2)
+
+	// should be same!
+	assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
+
+	// deserialize
+	m2, err := NewFromProto(&buf)
+	assert.Nil(t, err)
+	assert.NotNil(t, m2)
+
+	log.Dump(m2)
+}

mfer/gen.go

@@ -1,3 +0,0 @@
-package mfer
-
-//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto

mfer/manifest.go

@@ -1,35 +1,42 @@
 package mfer
 
 import (
+	"bytes"
+	"context"
+	"errors"
 	"fmt"
-	"io"
 	"io/fs"
-	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
+	"git.eeqj.de/sneak/mfer/internal/log"
 	"github.com/spf13/afero"
 )
 
-type ManifestFile struct {
-	Path     string
-	FileInfo fs.FileInfo
+type manifestFile struct {
+	path string
+	info fs.FileInfo
 }
 
-func (m *ManifestFile) String() string {
-	return fmt.Sprintf("<File \"%s\">", m.Path)
+func (m *manifestFile) String() string {
+	return fmt.Sprintf("<File \"%s\">", m.path)
 }
 
-type Manifest struct {
-	SourceFS      afero.Fs
-	SourceFSRoot  string
-	Files         []*ManifestFile
-	ScanOptions   *ManifestScanOptions
-	TotalFileSize int64
+type manifest struct {
+	sourceFS      []afero.Fs
+	files         []*manifestFile
+	scanOptions   *ManifestScanOptions
+	totalFileSize int64
+	pbInner       *MFFile
+	pbOuter       *MFFileOuter
+	output        *bytes.Buffer
+	ctx           context.Context
+	errors        []*error
 }
 
-func (m *Manifest) String() string {
-	return fmt.Sprintf("<Manifest count=%d totalSize=%d>", len(m.Files), m.TotalFileSize)
+func (m *manifest) String() string {
+	return fmt.Sprintf("<Manifest count=%d totalSize=%d>", len(m.files), m.totalFileSize)
 }
 
 type ManifestScanOptions struct {
@@ -37,78 +44,130 @@ type ManifestScanOptions struct {
 	FollowSymLinks bool
 }
 
-func NewFromPath(inputPath string, options *ManifestScanOptions) (*Manifest, error) {
+func (m *manifest) HasError() bool {
+	return len(m.errors) > 0
+}
+
+func (m *manifest) AddError(e error) *manifest {
+	m.errors = append(m.errors, &e)
+	return m
+}
+
+func (m *manifest) WithContext(c context.Context) *manifest {
+	m.ctx = c
+	return m
+}
+
+func (m *manifest) addInputPath(inputPath string) error {
 	abs, err := filepath.Abs(inputPath)
-	if err != nil {
-		return nil, err
-	}
-	afs := afero.NewBasePathFs(afero.NewOsFs(), abs)
-	m, err := NewFromFS(afs, options)
-	if err != nil {
-		return nil, err
-	}
-	m.SourceFSRoot = abs
-	return m, nil
-}
-
-func NewFromFS(fs afero.Fs, options *ManifestScanOptions) (*Manifest, error) {
-	m := &Manifest{
-		SourceFS:    fs,
-		ScanOptions: options,
-	}
-	err := m.Scan()
-	if err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (m *Manifest) Scan() error {
-	// FIXME scan and whatever function does the hashing should take ctx
-	oe := afero.Walk(m.SourceFS, "./", func(path string, info fs.FileInfo, err error) error {
-		if m.ScanOptions.IgnoreDotfiles && strings.HasPrefix(path, ".") {
-			// FIXME make this check all path components BUG
-			return nil
-		}
-
-		if info != nil && info.IsDir() {
-			// manifests contain only files, directories are implied.
-			return nil
-		}
-
-		fileinfo, staterr := m.SourceFS.Stat(path)
-		if staterr != nil {
-			panic(staterr)
-		}
-
-		nf := &ManifestFile{
-			Path:     path,
-			FileInfo: fileinfo,
-		}
-		m.Files = append(m.Files, nf)
-		m.TotalFileSize = m.TotalFileSize + info.Size()
-		return nil
-	})
-	if oe != nil {
-		return oe
-	}
-	return nil
-}
-
-func (m *Manifest) WriteToFile(path string) error {
-	// FIXME refuse to overwrite without -f if file exists
-
-	f, err := os.Create(path)
 	if err != nil {
 		return err
 	}
-	defer f.Close()
-
-	return m.Write(f)
+	// FIXME check to make sure inputPath/abs exists maybe
+	afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs))
+	return m.addInputFS(afs)
 }
 
-func (m *Manifest) Write(output io.Writer) error {
-	// FIXME implement
-	panic("nope")
-	return nil // nolint:all
+func (m *manifest) addInputFS(f afero.Fs) error {
+	if m.sourceFS == nil {
+		m.sourceFS = make([]afero.Fs, 0)
+	}
+	m.sourceFS = append(m.sourceFS, f)
+	// FIXME do some sort of check on f here?
+	return nil
+}
+
+func New() *manifest {
+	m := &manifest{}
+	return m
+}
+
+func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) {
+	log.Dump(inputPaths)
+	m := New()
+	m.scanOptions = options
+	for _, p := range inputPaths {
+		err := m.addInputPath(p)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return m, nil
+}
+
+func NewFromFS(options *ManifestScanOptions, fs afero.Fs) (*manifest, error) {
+	m := New()
+	m.scanOptions = options
+	err := m.addInputFS(fs)
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (m *manifest) GetFileCount() int64 {
+	return int64(len(m.files))
+}
+
+func (m *manifest) GetTotalFileSize() int64 {
+	return m.totalFileSize
+}
+
+func pathIsHidden(p string) bool {
+	tp := path.Clean(p)
+	if strings.HasPrefix(tp, ".") {
+		return true
+	}
+	for {
+		d, f := path.Split(tp)
+		if strings.HasPrefix(f, ".") {
+			return true
+		}
+		if d == "" {
+			return false
+		}
+		tp = d[0 : len(d)-1] // trim trailing slash from dir
+	}
+}
+
+func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
+	if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) {
+		return nil
+	}
+	if fi != nil && fi.IsDir() {
+		// manifests contain only files, directories are implied.
+		return nil
+	}
+	// FIXME test if 'fi' is already result of stat
+	fileinfo, staterr := m.sourceFS[sfsIndex].Stat(p)
+	if staterr != nil {
+		return staterr
+	}
+	cleanPath := p
+	if cleanPath[0:1] == "/" {
+		cleanPath = cleanPath[1:]
+	}
+	nf := &manifestFile{
+		path: cleanPath,
+		info: fileinfo,
+	}
+	m.files = append(m.files, nf)
+	m.totalFileSize = m.totalFileSize + fi.Size()
+	return nil
+}
+
+func (m *manifest) Scan() error {
+	// FIXME scan and whatever function does the hashing should take ctx
+	for idx, sfs := range m.sourceFS {
+		if sfs == nil {
+			return errors.New("invalid source fs")
+		}
+		e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error {
+			return m.addFile(p, info, idx)
+		})
+		if e != nil {
+			return e
+		}
+	}
+	return nil
 }

mfer/mf.proto

@@ -1,26 +1,34 @@
 syntax = "proto3";
 
-option go_package = "git.eeqj.de/sneak/mfer";
+option go_package = "git.eeqj.de/sneak/mfer/mfer";
 
 message Timestamp {
   int64 seconds = 1;
   int32 nanos = 2;
 }
 
-message MFFile {
+message MFFileOuter {
     enum Version {
-        NONE = 0;
-        ONE = 1; // only one for now
+        VERSION_NONE = 0;
+        VERSION_ONE = 1; // only one for now
     }
 
     // required mffile root attributes 1xx
     Version version = 101;
-    bytes innerMessage = 102;
+
+    enum CompressionType {
+        COMPRESSION_NONE = 0;
+        COMPRESSION_GZIP = 1;
+    }
+
+    CompressionType compressionType = 102;
+
     // these are used solely to detect corruption/truncation
     // and not for cryptographic integrity.
     int64 size = 103;
     bytes sha256 = 104;
 
+    bytes innerMessage = 199;
     // 2xx for optional manifest root attributes
     // think we might use gosignify instead of gpg:
     // github.com/frankbraun/gosignify
@@ -54,10 +62,10 @@ message MFFileChecksum {
     bytes multiHash = 1;
 }
 
-message MFFileInner {
+message MFFile {
     enum Version {
-        NONE = 0;
-        ONE = 1; // only one for now
+        VERSION_NONE = 0;
+        VERSION_ONE = 1; // only one for now
     }
     Version version = 100;
 

mfer/mfer_test.go

@@ -1,12 +1,74 @@
 package mfer
 
-import "testing"
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"git.eeqj.de/sneak/mfer/internal/log"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+)
 
 // Add those variables as well
 var (
-	existingFolder    = "./testdata/a-folder-that-exists"
-	nonExistingFolder = "./testdata/a-folder-that-does-not-exists"
+	existingFolder = "./testdata/a-folder-that-exists"
 )
 
-func TestManifestGeneration(t *testing.T) {
+var (
+	af  *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
+	big *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
+)
+
+func init() {
+	log.EnableDebugLogging()
+
+	// create test files and directories
+	af.MkdirAll("/a/b/c", 0o755)
+	af.MkdirAll("/.hidden", 0o755)
+	af.WriteFile("/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
+	af.WriteFile("/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
+	af.WriteFile("/.hidden/hello.txt", []byte("hello world\n"), 0o755)
+	af.WriteFile("/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
+
+	big.MkdirAll("/home/user/Library", 0o755)
+	for i := range [25]int{} {
+		big.WriteFile(fmt.Sprintf("/home/user/Library/hello%d.txt", i), []byte("hello world\n"), 0o755)
+	}
+}
+
+func TestPathHiddenFunc(t *testing.T) {
+	assert.False(t, pathIsHidden("/a/b/c/hello.txt"))
+	assert.True(t, pathIsHidden("/a/b/c/.hello.txt"))
+	assert.True(t, pathIsHidden("/a/.b/c/hello.txt"))
+	assert.True(t, pathIsHidden("/.a/b/c/hello.txt"))
+	assert.False(t, pathIsHidden("./a/b/c/hello.txt"))
+}
+
+func TestManifestGenerationOne(t *testing.T) {
+	m, err := NewFromFS(&ManifestScanOptions{
+		IgnoreDotfiles: true,
+	}, af)
+	assert.Nil(t, err)
+	assert.NotNil(t, m)
+	m.Scan()
+	assert.Equal(t, int64(2), m.GetFileCount())
+	assert.Equal(t, int64(30), m.GetTotalFileSize())
+}
+
+func TestManifestGenerationTwo(t *testing.T) {
+	m, err := NewFromFS(&ManifestScanOptions{
+		IgnoreDotfiles: false,
+	}, af)
+	assert.Nil(t, err)
+	assert.NotNil(t, m)
+	m.Scan()
+	assert.Equal(t, int64(4), m.GetFileCount())
+	assert.Equal(t, int64(54), m.GetTotalFileSize())
+	err = m.generate()
+	assert.Nil(t, err)
+	var buf bytes.Buffer
+	err = m.WriteTo(&buf)
+	assert.Nil(t, err)
+	log.Dump(buf.Bytes())
 }

mfer/output.go

@@ -0,0 +1,33 @@
+package mfer
+
+import (
+	"io"
+	"os"
+)
+
+func (m *manifest) WriteToFile(path string) error {
+	// FIXME refuse to overwrite without -f if file exists
+
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return m.WriteTo(f)
+}
+
+func (m *manifest) WriteTo(output io.Writer) error {
+	if m.pbOuter == nil {
+		err := m.generate()
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err := output.Write(m.output.Bytes())
+	if err != nil {
+		return err
+	}
+	return nil
+}

mfer/serialize.go

@@ -0,0 +1,100 @@
+package mfer
+
+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"errors"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+)
+
+//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
+
+// rot13("MANIFEST")
+const MAGIC string = "ZNAVSRFG"
+
+func newTimestampFromTime(t time.Time) *Timestamp {
+	out := &Timestamp{
+		Seconds: t.Unix(),
+		Nanos:   int32(t.UnixNano() - (t.Unix() * 1000000000)),
+	}
+	return out
+}
+
+func (m *manifest) generate() error {
+	if m.pbInner == nil {
+		e := m.generateInner()
+		if e != nil {
+			return e
+		}
+	}
+	if m.pbOuter == nil {
+		e := m.generateOuter()
+		if e != nil {
+			return e
+		}
+	}
+	dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
+	if err != nil {
+		return err
+	}
+	m.output = bytes.NewBuffer([]byte(MAGIC))
+	_, err = m.output.Write(dat)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *manifest) generateOuter() error {
+	if m.pbInner == nil {
+		return errors.New("internal error")
+	}
+	innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
+	if err != nil {
+		return err
+	}
+
+	h := sha256.New()
+	h.Write(innerData)
+
+	idc := new(bytes.Buffer)
+	gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
+	if err != nil {
+		return err
+	}
+	_, err = gzw.Write(innerData)
+	if err != nil {
+		return err
+	}
+
+	gzw.Close()
+
+	o := &MFFileOuter{
+		InnerMessage:    idc.Bytes(),
+		Size:            int64(len(innerData)),
+		Sha256:          h.Sum(nil),
+		Version:         MFFileOuter_VERSION_ONE,
+		CompressionType: MFFileOuter_COMPRESSION_GZIP,
+	}
+	m.pbOuter = o
+	return nil
+}
+
+func (m *manifest) generateInner() error {
+	m.pbInner = &MFFile{
+		Version:   MFFile_VERSION_ONE,
+		CreatedAt: newTimestampFromTime(time.Now()),
+		Files:     []*MFFilePath{},
+	}
+	for _, f := range m.files {
+		nf := &MFFilePath{
+			Path: f.path,
+			// FIXME add more stuff
+		}
+		m.pbInner.Files = append(m.pbInner.Files, nf)
+	}
+	return nil
+}