add link to styleguide

Reviewed-on: #8

manually approving because CI is offline rn for some reason

README.md

@@ -1,11 +1,50 @@
 # mfer
 
-Manifest file generator and checker.
+[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library
+and thin wrapper command-line utility written in [Go](https://golang.org)
+and first published in 2022 under the [WTFPL](https://wtfpl.net) (public
+domain) license.  It specifies and generates `.mf` manifest files over a
+directory tree of files to encapsulate metadata about them (such as
+cryptographic checksums or signatures over same) to aid in archiving,
+downloading, and streaming, or mirroring.  The manifest files' data is
+serialized with Google's [protobuf serialization
+format](https://developers.google.com/protocol-buffers).  The structure of
+these files can be found [in the format
+specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
+which is included in the [project
+repository](https://git.eeqj.de/sneak/mfer).
+
+The current version is pre-1.0 and while the repo was published in 2022,
+there has not yet been any versioned release.  [SemVer](https://semver.org)
+will be used for releases.
+
+This project was started by [@sneak](https://sneak.berlin) to scratch an
+itch in 2022 and is currently a one-person effort, though the goal is for
+this to emerge as a de-facto standard and be incorporated into other
+software.  A compatible javascript library is planned.
 
 # Build Status
 
 [![Build Status](https://drone.datavi.be/api/badges/sneak/mfer/status.svg)](https://drone.datavi.be/sneak/mfer)
 
+# Participation
+
+The community is as yet nonexistent so there are no defined policies or
+norms yet.  Primary development happens on a privately-run Gitea instance at
+[https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) and issues
+are [tracked there](https://git.eeqj.de/sneak/mfer/issues).
+
+Changes must always be formatted with a standard `go fmt`, syntactically
+valid, and must pass the linting defined in the repository (presently only
+the `golangci-lint` defaults), which can be run with a `make lint`.  The
+`main` branch is protected and all changes must be made via [pull
+requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
+Any changes submitted to this project must also be
+[WTFPL-licensed](https://wtfpl.net) to be considered.
+
+Ideally, contributions conform to @sneak's personal 
+[code styleguide](https://git.eeqj.de/sneak/styleguide).
+
 # Problem Statement
 
 Given a plain URL, there is no standard way to safely and programmatically
@@ -170,6 +209,15 @@ regardless of filesystem format.
 Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your
 desired username for an account on this Gitea instance.
 
-I am currently interested in hiring a contractor skilled with the Go
+## Links
-standard library interfaces to specify this tool in full and develop a
+
-prototype implementation.
+* Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
+* Issues: [https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
+
+# Authors
+
+* [@sneak <sneak@sneak.berlin>](mailto:sneak@sneak.berlin)
+
+# License
+
+* [WTFPL](https://wtfpl.net)

internal/bork/error.go

@@ -2,10 +2,14 @@ package bork
 
 import (
 	"errors"
+	"fmt"
 )
 
 var (
 	ErrMissingMagic  = errors.New("missing magic bytes in file")
 	ErrFileTruncated = errors.New("file/stream is truncated abnormally")
-	ErrFileIntegrity = errors.New("file/stream checksum failure")
 )
+
+func Newf(format string, args ...interface{}) error {
+	return fmt.Errorf(format, args...)
+}

internal/log/log.go

@@ -23,20 +23,11 @@ func DisableStyling() {
 	pterm.Fatal.Prefix.Text = ""
 }
 
-var TestingTraces bool = false
-
 func Init() {
 	log.SetHandler(acli.Default)
 	log.SetLevel(log.InfoLevel)
 }
 
-func Tracef(format string, args ...interface{}) {
-	if !TestingTraces {
-		return
-	}
-	DebugReal(fmt.Sprintf(format, args...), 2)
-}
-
 func Debugf(format string, args ...interface{}) {
 	DebugReal(fmt.Sprintf(format, args...), 2)
 }
@@ -54,22 +45,10 @@ func DebugReal(arg string, cs int) {
 	log.Debug(tag + arg)
 }
 
-func TraceDump(args ...interface{}) {
-	if !TestingTraces {
-		return
-	}
-	DebugReal(spew.Sdump(args...), 2)
-}
-
 func Dump(args ...interface{}) {
 	DebugReal(spew.Sdump(args...), 2)
 }
 
-func EnableTestingLogging() {
-	TestingTraces = true
-	EnableDebugLogging()
-}
-
 func EnableDebugLogging() {
 	SetLevel(log.DebugLevel)
 }

mfer/deserialize.go

@@ -3,7 +3,6 @@ package mfer
 import (
 	"bytes"
 	"compress/gzip"
-	"crypto/sha256"
 	"errors"
 	"io"
 
@@ -12,35 +11,12 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
-func (m *manifest) validateProtoInner() error {
-	i := m.pbInner
-	if i.Version != MFFile_VERSION_ONE {
-		return errors.New("unknown version") // FIXME move to bork
-	}
-
-	if len(i.Files) == 0 {
-		return errors.New("manifest without files") // FIXME move to bork
-	}
-
-	for _, mfp := range m.pbInner.Files {
-		// there is no way we should be doing validateProtoInner()
-		// outside of a load into a blank/empty/new *manifest
-		if m.files != nil {
-			return errors.New("shouldn't happen, internal error")
-		}
-		m.files = make([]*manifestFile, 0)
-		// we can skip error handling here thanks to the magic of protobuf
-		m.addFileLoadTime(mfp)
-	}
-	return nil
-}
-
 func (m *manifest) validateProtoOuter() error {
 	if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
-		return errors.New("unknown version") // FIXME move to bork
+		return errors.New("unknown version")
 	}
 	if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
-		return errors.New("unknown compression type") // FIXME move to bork
+		return errors.New("unknown compression type")
 	}
 
 	bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
@@ -59,26 +35,12 @@ func (m *manifest) validateProtoOuter() error {
 
 	isize := len(dat)
 	if int64(isize) != m.pbOuter.Size {
-		log.Tracef("truncated data, got %d expected %d", isize, m.pbOuter.Size)
+		log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
 		return bork.ErrFileTruncated
 	}
-	log.Tracef("inner data size is %d", isize)
+	log.Debugf("inner data size is %d", isize)
-	log.TraceDump(dat)
+	log.Dump(dat)
-
+	log.Dump(m.pbOuter.Sha256)
-	// FIXME validate Sha256
-	log.TraceDump(m.pbOuter.Sha256)
-
-	h := sha256.New()
-	h.Write(dat)
-	shaGot := h.Sum(nil)
-
-	log.TraceDump("got: ", shaGot)
-	log.TraceDump("expected: ", m.pbOuter.Sha256)
-
-	if !bytes.Equal(shaGot, m.pbOuter.Sha256) {
-		m.pbOuter.InnerMessage = nil // don't try to mess with it
-		return bork.ErrFileIntegrity
-	}
 	return nil
 }
 
@@ -92,7 +54,7 @@ func validateMagic(dat []byte) bool {
 	return bytes.Equal(got, expected)
 }
 
-func NewFromReader(input io.Reader) (*manifest, error) {
+func NewFromProto(input io.Reader) (*manifest, error) {
 	m := New()
 	dat, err := io.ReadAll(input)
 	if err != nil {
@@ -107,7 +69,7 @@ func NewFromReader(input io.Reader) (*manifest, error) {
 	bb := bytes.NewBuffer(dat[ml:])
 	dat = bb.Bytes()
 
-	log.TraceDump(dat)
+	log.Dump(dat)
 
 	// deserialize:
 	m.pbOuter = new(MFFileOuter)
@@ -122,22 +84,6 @@ func NewFromReader(input io.Reader) (*manifest, error) {
 		return nil, ve
 	}
 
-	m.pbInner = new(MFFile)
+	// FIXME TODO deserialize inner
-	err = proto.Unmarshal(m.pbOuter.InnerMessage, m.pbInner)
-	if err != nil {
-		return nil, err
-	}
-
-	log.TraceDump(m.pbInner)
-
-	ve = m.validateProtoInner()
-	if ve != nil {
-		return nil, ve
-	}
-
-	m.rescanInternal()
-
-	log.TraceDump(m)
-
 	return m, nil
 }

mfer/example_test.go

@@ -8,10 +8,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func init() {
-	log.EnableTestingLogging()
-}
-
 func TestAPIExample(t *testing.T) {
 	// read from filesystem
 	m, err := NewFromFS(&ManifestScanOptions{
@@ -28,7 +24,7 @@ func TestAPIExample(t *testing.T) {
 	m.WriteTo(&buf)
 
 	// show serialized
-	log.TraceDump(buf.Bytes())
+	log.Dump(buf.Bytes())
 
 	// do it again
 	var buf2 bytes.Buffer
@@ -38,9 +34,9 @@ func TestAPIExample(t *testing.T) {
 	assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
 
 	// deserialize
-	m2, err := NewFromReader(&buf)
+	m2, err := NewFromProto(&buf)
 	assert.Nil(t, err)
 	assert.NotNil(t, m2)
 
-	log.TraceDump(m2)
+	log.Dump(m2)
 }

mfer/manifest.go

@@ -17,8 +17,6 @@ import (
 type manifestFile struct {
 	path string
 	info fs.FileInfo
-	size      int64
-	protoFile *MFFilePath
 }
 
 func (m *manifestFile) String() string {
@@ -85,7 +83,7 @@ func New() *manifest {
 }
 
 func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) {
-	log.TraceDump(inputPaths)
+	log.Dump(inputPaths)
 	m := New()
 	m.scanOptions = options
 	for _, p := range inputPaths {
@@ -111,10 +109,6 @@ func (m *manifest) GetFileCount() int64 {
 	return int64(len(m.files))
 }
 
-func (m *manifest) rescanInternal() {
-	panic("unimplemented")
-}
-
 func (m *manifest) GetTotalFileSize() int64 {
 	return m.totalFileSize
 }
@@ -136,17 +130,7 @@ func pathIsHidden(p string) bool {
 	}
 }
 
-func (m *manifest) addFileLoadTime(in *MFFilePath) {
+func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
-	nf := &manifestFile{
-		path:      in.Path,
-		size:      in.Size,
-		protoFile: in, // FIXME get rid of this eventually
-	}
-	m.files = append(m.files, nf)
-	m.totalFileSize = m.totalFileSize + in.Size
-}
-
-func (m *manifest) addFileScanTime(p string, fi fs.FileInfo, sfsIndex int) error {
 	if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) {
 		return nil
 	}
@@ -179,7 +163,7 @@ func (m *manifest) Scan() error {
 			return errors.New("invalid source fs")
 		}
 		e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error {
-			return m.addFileScanTime(p, info, idx)
+			return m.addFile(p, info, idx)
 		})
 		if e != nil {
 			return e

mfer/mf.proto

@@ -9,7 +9,7 @@ message Timestamp {
 
 message MFFileOuter {
     enum Version {
-        VERSION_NONE = 0; // must have a zero enum by law
+        VERSION_NONE = 0;
         VERSION_ONE = 1; // only one for now
     }
 
@@ -17,7 +17,7 @@ message MFFileOuter {
     Version version = 101;
 
     enum CompressionType {
-        COMPRESSION_NONE = 0; // must have a zero enum by law
+        COMPRESSION_NONE = 0;
         COMPRESSION_GZIP = 1;
     }
 
@@ -33,10 +33,12 @@ message MFFileOuter {
     // think we might use gosignify instead of gpg:
     // github.com/frankbraun/gosignify
 
-    //detached signatures, ascii or binary
+    //detached signature, ascii or binary
-    repeated bytes signatures = 201;
+    optional bytes signature = 201;
-    //full GPG signing public keys, ascii or binary
+    //full GPG key id
-    repeated bytes signingPubKeys = 203;
+    optional bytes signer = 202;
+    //full GPG signing public key, ascii or binary
+    optional bytes signingPubKey = 203;
 }
 
 message MFFilePath {
@@ -56,13 +58,13 @@ message MFFilePath {
 
 message MFFileChecksum {
     // 1.0 golang implementation must write a multihash here
-    // it's ok to only ever use/verify sha256 multihash i think?
+    // it's ok to only ever use/verify sha256 multihash
     bytes multiHash = 1;
 }
 
 message MFFile {
     enum Version {
-        VERSION_NONE = 0;  // must have a zero enum by law
+        VERSION_NONE = 0;
         VERSION_ONE = 1; // only one for now
     }
     Version version = 100;

mfer/mfer_test.go

@@ -21,7 +21,7 @@ var (
 )
 
 func init() {
-	log.EnableTestingLogging()
+	log.EnableDebugLogging()
 
 	// create test files and directories
 	af.MkdirAll("/a/b/c", 0o755)
@@ -70,5 +70,5 @@ func TestManifestGenerationTwo(t *testing.T) {
 	var buf bytes.Buffer
 	err = m.WriteTo(&buf)
 	assert.Nil(t, err)
-	log.TraceDump(buf.Bytes())
+	log.Dump(buf.Bytes())
 }

mfer/serialize.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"time"
 
-	"git.eeqj.de/sneak/mfer/internal/log"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -60,7 +59,6 @@ func (m *manifest) generateOuter() error {
 
 	h := sha256.New()
 	h.Write(innerData)
-	sha256 := h.Sum(nil)
 
 	idc := new(bytes.Buffer)
 	gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
@@ -74,12 +72,10 @@ func (m *manifest) generateOuter() error {
 
 	gzw.Close()
 
-	log.Tracef("calculated sha256 (uncompressed): %x\n", sha256)
-
 	o := &MFFileOuter{
 		InnerMessage:    idc.Bytes(),
 		Size:            int64(len(innerData)),
-		Sha256:          sha256,
+		Sha256:          h.Sum(nil),
 		Version:         MFFileOuter_VERSION_ONE,
 		CompressionType: MFFileOuter_COMPRESSION_GZIP,
 	}