latest

.dockerignore

@@ -1,7 +1,3 @@
-*.tzst
-*.tar
-/buildimage
-/dockerdeps
-/tmp
-*.docker.tzst
 *.tmp
+*.dockerimage
+.git

.drone.yml

@@ -7,8 +7,17 @@ steps:
   network_mode: bridge
   settings:
     repo: sneak/mfer
+    build_args_from_env: [ DRONE_COMMIT_SHA ]
     dry_run: true
     custom_dns: [ 116.202.204.30 ]
     tags:
-      - ${DRONE_COMMIT_SHA}
+      - ${DRONE_COMMIT_SHA:0:7}
       - ${DRONE_BRANCH}
+      - latest
+- name: notify
+  image: plugins/slack
+  settings:
+    webhook:
+      from_secret: SLACK_WEBHOOK_URL
+  when:
+    event: pull_request

.gitignore

@@ -1,8 +1,6 @@
 mfer/*.pb.go
 /mfer.cmd
-vendor
 /tmp
 *.tmp
-*.docker.tzst
+*.dockerimage
-*.tzst
+/vendor
-/builddeps/modcache.tar

Dockerfile

@@ -1,29 +1,30 @@
 ################################################################################
 #2345678911234567892123456789312345678941234567895123456789612345678971234567898
 ################################################################################
-FROM sneak/builder:main AS builder
+FROM sneak/builder:2022-12-08 AS builder
-ENV GOPATH /go
 ENV DEBIAN_FRONTEND noninteractive
 WORKDIR /build
-COPY ./go.mod ./go.sum .
+COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum /build/
-RUN \
+COPY ./vendor.tzst /build/vendor.tzst
-    go mod download -x
+COPY ./modcache.tzst /build/modcache.tzst
-################################################################################
-#### caching phase done
-################################################################################
-WORKDIR /build
-COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum .
 COPY ./internal ./internal
 COPY ./bin/gitrev.sh ./bin/gitrev.sh
 COPY ./mfer ./mfer
 COPY ./cmd ./cmd
-RUN find /build
 ARG GITREV unknown
+ARG DRONE_COMMIT_SHA unknown
+
+RUN mkdir -p "$(go env GOMODCACHE)" && cd "$(go env GOMODCACHE)" && \
+    zstdmt -d --stdout /build/modcache.tzst | tar xf - && \
+    rm /build/modcache.tzst && cd /build
 RUN \
     cd mfer && go generate . && cd .. && \
     GOPACKAGESDEBUG=true golangci-lint run ./... && \
+    mkdir vendor && cd vendor && \
+    zstdmt -d --stdout /build/vendor.tzst | tar xf - && rm /build/vendor.tzst && \
+    cd .. && \
     make mfer.cmd
-RUN go mod vendor && tar -c . | zstdmt -19 > /src.tzst
+RUN rm -rf /build/vendor && go mod vendor && tar -c . | zstdmt -19 > /src.tzst
 ################################################################################
 #2345678911234567892123456789312345678941234567895123456789612345678971234567898
 ################################################################################

Makefile

@@ -44,9 +44,9 @@ mfer.cmd: $(SOURCEFILES) mfer/mf.pb.go
 	cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../mfer.cmd $(GOFLAGS) .
 
 clean:
-	rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer
+	rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer *.dockerimage
 
-fmt:
+fmt: mfer/mf.pb.go
 	gofumpt -l -w mfer internal cmd
 	golangci-lint run --fix
 	-prettier -w *.json
@@ -56,9 +56,9 @@ lint:
 	golangci-lint run
 	sh -c 'test -z "$$(gofmt -l .)"'
 
-docker: sneak-mfer.$(ARCH).docker.tzst
+docker: sneak-mfer.$(ARCH).tzst.dockerimage
 
-sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES)
+sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
 	docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
 	docker save sneak/mfer | pv | zstdmt -19 > $@
 	du -sh $@
@@ -66,3 +66,17 @@ sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES)
 godoc:
 	open http://127.0.0.1:6060
 	godoc -http=:6060
+
+vendor.tzst: go.mod go.sum
+	go mod tidy
+	go mod vendor
+	cd vendor && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
+	rm -rf vendor
+	mv $@.tmp $@
+
+modcache.tzst: go.mod go.sum
+	go mod tidy
+	cd $(HOME)/go/pkg && chmod -R u+rw . && rm -rf mod sumdb
+	go mod download -x
+	cd $(shell go env GOMODCACHE) && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
+	mv $@.tmp $@

bin/gitrev.sh

@@ -1,4 +1,10 @@
 #!/bin/bash
+#
+if [[ ! -z "$DRONE_COMMIT_SHA" ]]; then
+    echo "${DRONE_COMMIT_SHA:0:7}"
+    exit 0
+fi
+
 if [[ ! -z "$GITREV" ]]; then
     echo $GITREV
 else

buildimage/Dockerfile

@@ -1,21 +0,0 @@
-## build image:
-FROM golang:1.19.3-bullseye AS builder
-
-ENV DEBIAN_FRONTEND noninteractive
-RUN apt update && apt install -y make bzip2 curl unzip
-RUN mkdir -p /build
-WORKDIR /build
-
-# install newer protoc
-RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \
-    unzip *.zip -d /usr/local && rm -v *.zip && protoc --version
-
-RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
-
-RUN go env
-
-COPY ./go.mod .
-COPY ./go.sum .
-
-RUN --mount=type=cache,target=/go/pkg go mod download -x
-RUN rm -rfv /var/cache/* /var/tmp/*

internal/bork/error.go

@@ -2,12 +2,10 @@ package bork
 
 import (
 	"errors"
-	"fmt"
 )
 
-var ErrMissingMagic = errors.New("missing magic bytes in file")
+var (
-var ErrFileTruncated = errors.New("file/stream is truncated abnormally")
+	ErrMissingMagic  = errors.New("missing magic bytes in file")
-
+	ErrFileTruncated = errors.New("file/stream is truncated abnormally")
-func Newf(format string, args ...interface{}) error {
+	ErrFileIntegrity = errors.New("file/stream checksum failure")
-	return errors.New(fmt.Sprintf(format, args...))
+)
-}

internal/bork/error_test.go

@@ -0,0 +1,11 @@
+package bork
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuild(t *testing.T) {
+	assert.NotNil(t, ErrMissingMagic)
+}

internal/cli/misc.go

@@ -1,32 +0,0 @@
-package cli
-
-import "fmt"
-
-// FIXME make this write to a bytes.Buffer with fprintf
-func DumpByteSlice(b []byte) {
-	var a [16]byte
-	n := (len(b) + 15) &^ 15
-	for i := 0; i < n; i++ {
-		if i%16 == 0 {
-			fmt.Printf("%4d", i)
-		}
-		if i%8 == 0 {
-			fmt.Print(" ")
-		}
-		if i < len(b) {
-			fmt.Printf(" %02X", b[i])
-		} else {
-			fmt.Print("   ")
-		}
-		if i >= len(b) {
-			a[i%16] = ' '
-		} else if b[i] < 32 || b[i] > 126 {
-			a[i%16] = '.'
-		} else {
-			a[i%16] = b[i]
-		}
-		if i%16 == 15 {
-			fmt.Printf("  %s\n", string(a[:]))
-		}
-	}
-}

internal/log/log.go

@@ -23,11 +23,20 @@ func DisableStyling() {
 	pterm.Fatal.Prefix.Text = ""
 }
 
+var TestingTraces bool = false
+
 func Init() {
 	log.SetHandler(acli.Default)
 	log.SetLevel(log.InfoLevel)
 }
 
+func Tracef(format string, args ...interface{}) {
+	if !TestingTraces {
+		return
+	}
+	DebugReal(fmt.Sprintf(format, args...), 2)
+}
+
 func Debugf(format string, args ...interface{}) {
 	DebugReal(fmt.Sprintf(format, args...), 2)
 }
@@ -45,10 +54,22 @@ func DebugReal(arg string, cs int) {
 	log.Debug(tag + arg)
 }
 
+func TraceDump(args ...interface{}) {
+	if !TestingTraces {
+		return
+	}
+	DebugReal(spew.Sdump(args...), 2)
+}
+
 func Dump(args ...interface{}) {
 	DebugReal(spew.Sdump(args...), 2)
 }
 
+func EnableTestingLogging() {
+	TestingTraces = true
+	EnableDebugLogging()
+}
+
 func EnableDebugLogging() {
 	SetLevel(log.DebugLevel)
 }

internal/log/log_test.go

@@ -0,0 +1,12 @@
+package log
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuild(t *testing.T) {
+	Init()
+	assert.True(t, true)
+}

mfer/deserialize.go

@@ -3,6 +3,7 @@ package mfer
 import (
 	"bytes"
 	"compress/gzip"
+	"crypto/sha256"
 	"errors"
 	"io"
 
@@ -11,12 +12,35 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+func (m *manifest) validateProtoInner() error {
+	i := m.pbInner
+	if i.Version != MFFile_VERSION_ONE {
+		return errors.New("unknown version") // FIXME move to bork
+	}
+
+	if len(i.Files) == 0 {
+		return errors.New("manifest without files") // FIXME move to bork
+	}
+
+	for _, mfp := range m.pbInner.Files {
+		// there is no way we should be doing validateProtoInner()
+		// outside of a load into a blank/empty/new *manifest
+		if m.files != nil {
+			return errors.New("shouldn't happen, internal error")
+		}
+		m.files = make([]*manifestFile, 0)
+		// we can skip error handling here thanks to the magic of protobuf
+		m.addFileLoadTime(mfp)
+	}
+	return nil
+}
+
 func (m *manifest) validateProtoOuter() error {
 	if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
-		return errors.New("unknown version")
+		return errors.New("unknown version") // FIXME move to bork
 	}
 	if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
-		return errors.New("unknown compression type")
+		return errors.New("unknown compression type") // FIXME move to bork
 	}
 
 	bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
@@ -35,12 +59,26 @@ func (m *manifest) validateProtoOuter() error {
 
 	isize := len(dat)
 	if int64(isize) != m.pbOuter.Size {
-		log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
+		log.Tracef("truncated data, got %d expected %d", isize, m.pbOuter.Size)
 		return bork.ErrFileTruncated
 	}
-	log.Debugf("inner data size is %d", isize)
+	log.Tracef("inner data size is %d", isize)
-	log.Dump(dat)
+	log.TraceDump(dat)
-	log.Dump(m.pbOuter.Sha256)
+
+	// FIXME validate Sha256
+	log.TraceDump(m.pbOuter.Sha256)
+
+	h := sha256.New()
+	h.Write(dat)
+	shaGot := h.Sum(nil)
+
+	log.TraceDump("got: ", shaGot)
+	log.TraceDump("expected: ", m.pbOuter.Sha256)
+
+	if !bytes.Equal(shaGot, m.pbOuter.Sha256) {
+		m.pbOuter.InnerMessage = nil // don't try to mess with it
+		return bork.ErrFileIntegrity
+	}
 	return nil
 }
 
@@ -54,7 +92,7 @@ func validateMagic(dat []byte) bool {
 	return bytes.Equal(got, expected)
 }
 
-func NewFromProto(input io.Reader) (*manifest, error) {
+func NewFromReader(input io.Reader) (*manifest, error) {
 	m := New()
 	dat, err := io.ReadAll(input)
 	if err != nil {
@@ -69,7 +107,7 @@ func NewFromProto(input io.Reader) (*manifest, error) {
 	bb := bytes.NewBuffer(dat[ml:])
 	dat = bb.Bytes()
 
-	log.Dump(dat)
+	log.TraceDump(dat)
 
 	// deserialize:
 	m.pbOuter = new(MFFileOuter)
@@ -84,6 +122,22 @@ func NewFromProto(input io.Reader) (*manifest, error) {
 		return nil, ve
 	}
 
-	// FIXME TODO deserialize inner
+	m.pbInner = new(MFFile)
+	err = proto.Unmarshal(m.pbOuter.InnerMessage, m.pbInner)
+	if err != nil {
+		return nil, err
+	}
+
+	log.TraceDump(m.pbInner)
+
+	ve = m.validateProtoInner()
+	if ve != nil {
+		return nil, ve
+	}
+
+	m.rescanInternal()
+
+	log.TraceDump(m)
+
 	return m, nil
 }

mfer/example_test.go

@@ -8,8 +8,11 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestAPIExample(t *testing.T) {
+func init() {
+	log.EnableTestingLogging()
+}
 
+func TestAPIExample(t *testing.T) {
 	// read from filesystem
 	m, err := NewFromFS(&ManifestScanOptions{
 		IgnoreDotfiles: true,
@@ -25,7 +28,7 @@ func TestAPIExample(t *testing.T) {
 	m.WriteTo(&buf)
 
 	// show serialized
-	log.Dump(buf.Bytes())
+	log.TraceDump(buf.Bytes())
 
 	// do it again
 	var buf2 bytes.Buffer
@@ -35,9 +38,9 @@ func TestAPIExample(t *testing.T) {
 	assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
 
 	// deserialize
-	m2, err := NewFromProto(&buf)
+	m2, err := NewFromReader(&buf)
 	assert.Nil(t, err)
 	assert.NotNil(t, m2)
 
-	log.Dump(m2)
+	log.TraceDump(m2)
 }

mfer/manifest.go

@@ -15,8 +15,10 @@ import (
 )
 
 type manifestFile struct {
-	path string
+	path      string
-	info fs.FileInfo
+	info      fs.FileInfo
+	size      int64
+	protoFile *MFFilePath
 }
 
 func (m *manifestFile) String() string {
@@ -83,7 +85,7 @@ func New() *manifest {
 }
 
 func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) {
-	log.Dump(inputPaths)
+	log.TraceDump(inputPaths)
 	m := New()
 	m.scanOptions = options
 	for _, p := range inputPaths {
@@ -109,6 +111,10 @@ func (m *manifest) GetFileCount() int64 {
 	return int64(len(m.files))
 }
 
+func (m *manifest) rescanInternal() {
+	panic("unimplemented")
+}
+
 func (m *manifest) GetTotalFileSize() int64 {
 	return m.totalFileSize
 }
@@ -130,7 +136,17 @@ func pathIsHidden(p string) bool {
 	}
 }
 
-func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
+func (m *manifest) addFileLoadTime(in *MFFilePath) {
+	nf := &manifestFile{
+		path:      in.Path,
+		size:      in.Size,
+		protoFile: in, // FIXME get rid of this eventually
+	}
+	m.files = append(m.files, nf)
+	m.totalFileSize = m.totalFileSize + in.Size
+}
+
+func (m *manifest) addFileScanTime(p string, fi fs.FileInfo, sfsIndex int) error {
 	if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) {
 		return nil
 	}
@@ -163,7 +179,7 @@ func (m *manifest) Scan() error {
 			return errors.New("invalid source fs")
 		}
 		e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error {
-			return m.addFile(p, info, idx)
+			return m.addFileScanTime(p, info, idx)
 		})
 		if e != nil {
 			return e

mfer/mf.proto

@@ -9,7 +9,7 @@ message Timestamp {
 
 message MFFileOuter {
     enum Version {
-        VERSION_NONE = 0;
+        VERSION_NONE = 0; // must have a zero enum by law
         VERSION_ONE = 1; // only one for now
     }
 
@@ -17,7 +17,7 @@ message MFFileOuter {
     Version version = 101;
 
     enum CompressionType {
-        COMPRESSION_NONE = 0;
+        COMPRESSION_NONE = 0; // must have a zero enum by law
         COMPRESSION_GZIP = 1;
     }
 
@@ -33,12 +33,10 @@ message MFFileOuter {
     // think we might use gosignify instead of gpg:
     // github.com/frankbraun/gosignify
 
-    //detached signature, ascii or binary
+    //detached signatures, ascii or binary
-    optional bytes signature = 201;
+    repeated bytes signatures = 201;
-    //full GPG key id
+    //full GPG signing public keys, ascii or binary
-    optional bytes signer = 202;
+    repeated bytes signingPubKeys = 203;
-    //full GPG signing public key, ascii or binary
-    optional bytes signingPubKey = 203;
 }
 
 message MFFilePath {
@@ -58,13 +56,13 @@ message MFFilePath {
 
 message MFFileChecksum {
     // 1.0 golang implementation must write a multihash here
-    // it's ok to only ever use/verify sha256 multihash
+    // it's ok to only ever use/verify sha256 multihash i think?
     bytes multiHash = 1;
 }
 
 message MFFile {
     enum Version {
-        VERSION_NONE = 0;
+        VERSION_NONE = 0;  // must have a zero enum by law
         VERSION_ONE = 1; // only one for now
     }
     Version version = 100;

mfer/mfer_test.go

@@ -21,7 +21,7 @@ var (
 )
 
 func init() {
-	log.EnableDebugLogging()
+	log.EnableTestingLogging()
 
 	// create test files and directories
 	af.MkdirAll("/a/b/c", 0o755)
@@ -32,7 +32,7 @@ func init() {
 	af.WriteFile("/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
 
 	big.MkdirAll("/home/user/Library", 0o755)
-	for i, _ := range [25]int{} {
+	for i := range [25]int{} {
 		big.WriteFile(fmt.Sprintf("/home/user/Library/hello%d.txt", i), []byte("hello world\n"), 0o755)
 	}
 }
@@ -70,5 +70,5 @@ func TestManifestGenerationTwo(t *testing.T) {
 	var buf bytes.Buffer
 	err = m.WriteTo(&buf)
 	assert.Nil(t, err)
-	log.Dump(buf.Bytes())
+	log.TraceDump(buf.Bytes())
 }

mfer/serialize.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"time"
 
+	"git.eeqj.de/sneak/mfer/internal/log"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -59,6 +60,7 @@ func (m *manifest) generateOuter() error {
 
 	h := sha256.New()
 	h.Write(innerData)
+	sha256 := h.Sum(nil)
 
 	idc := new(bytes.Buffer)
 	gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
@@ -72,10 +74,12 @@ func (m *manifest) generateOuter() error {
 
 	gzw.Close()
 
+	log.Tracef("calculated sha256 (uncompressed): %x\n", sha256)
+
 	o := &MFFileOuter{
 		InnerMessage:    idc.Bytes(),
 		Size:            int64(len(innerData)),
-		Sha256:          h.Sum(nil),
+		Sha256:          sha256,
 		Version:         MFFileOuter_VERSION_ONE,
 		CompressionType: MFFileOuter_COMPRESSION_GZIP,
 	}