latest

Dockerfile

@@ -1,27 +1,9 @@
 ################################################################################
 #2345678911234567892123456789312345678941234567895123456789612345678971234567898
 ################################################################################
-# Lint stage — fast feedback on formatting and lint issues
+FROM sneak/builder:2022-12-08 AS builder
-# golangci/golangci-lint:v1.64.8
-FROM golangci/golangci-lint@sha256:2987913e27f4eca9c8a39129d2c7bc1e74fbcf77f181e01cea607be437aa5cb8 AS lint
-WORKDIR /src
-RUN apt-get update && apt-get install -y --no-install-recommends protobuf-compiler && rm -rf /var/lib/apt/lists/*
-COPY go.mod go.sum ./
-RUN go mod download
-RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
-COPY . .
-RUN cd mfer && go generate .
-RUN make lint
-################################################################################
-#2345678911234567892123456789312345678941234567895123456789612345678971234567898
-################################################################################
-# Build stage
-# sneak/builder:2022-12-08
-FROM sneak/builder@sha256:d61175320b02b04eb2cada3005833494ce8a4032647e4202dcb3551cf0b56f5a AS builder
 ENV DEBIAN_FRONTEND noninteractive
 WORKDIR /build
-# Force BuildKit to run the lint stage by creating a stage dependency
-COPY --from=lint /src/go.sum /dev/null
 COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum /build/
 COPY ./vendor.tzst /build/vendor.tzst
 COPY ./modcache.tzst /build/modcache.tzst
@@ -37,6 +19,7 @@ RUN mkdir -p "$(go env GOMODCACHE)" && cd "$(go env GOMODCACHE)" && \
     rm /build/modcache.tzst && cd /build
 RUN \
     cd mfer && go generate . && cd .. && \
+    GOPACKAGESDEBUG=true golangci-lint run ./... && \
     mkdir vendor && cd vendor && \
     zstdmt -d --stdout /build/vendor.tzst | tar xf - && rm /build/vendor.tzst && \
     cd .. && \

README.md

@@ -1,48 +1,11 @@
 # mfer
 
-[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library
+Manifest file generator and checker.
-and thin wrapper command-line utility written in [Go](https://golang.org)
-and first published in 2022 under the [WTFPL](https://wtfpl.net) (public
-domain) license.  It specifies and generates `.mf` manifest files over a
-directory tree of files to encapsulate metadata about them (such as
-cryptographic checksums or signatures over same) to aid in archiving,
-downloading, and streaming, or mirroring.  The manifest files' data is
-serialized with Google's [protobuf serialization
-format](https://developers.google.com/protocol-buffers).  The structure of
-these files can be found [in the format
-specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
-which is included in the [project
-repository](https://git.eeqj.de/sneak/mfer).
-
-The current version is pre-1.0 and while the repo was published in 2022,
-there has not yet been any versioned release.  [SemVer](https://semver.org)
-will be used for releases.
-
-This project was started by [@sneak](https://sneak.berlin) to scratch an
-itch in 2022 and is currently a one-person effort, though the goal is for
-this to emerge as a de-facto standard and be incorporated into other
-software.  A compatible javascript library is planned.
 
 # Build Status
 
 [![Build Status](https://drone.datavi.be/api/badges/sneak/mfer/status.svg)](https://drone.datavi.be/sneak/mfer)
 
-# Participation
-
-The community is as yet nonexistent so there are no defined policies or
-norms yet.  Primary development happens on a privately-run Gitea instance at
-[https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) and issues
-are [tracked there](https://git.eeqj.de/sneak/mfer/issues).
-
-Changes must always be formatted with a standard `go fmt`, syntactically
-valid, and must pass the linting defined in the repository (presently only
-the `golangci-lint` defaults), which can be run with a `make lint`.  The
-`main` branch is protected and all changes must be made via [pull
-requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
-Any changes submitted to this project must also be
-[WTFPL-licensed](https://wtfpl.net) to be considered.
-
-
 # Problem Statement
 
 Given a plain URL, there is no standard way to safely and programmatically
@@ -207,24 +170,6 @@ regardless of filesystem format.
 Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your
 desired username for an account on this Gitea instance.
 
-# See Also
+I am currently interested in hiring a contractor skilled with the Go
-
+standard library interfaces to specify this tool in full and develop a
-## Prior Art: Metalink
+prototype implementation.
-
-* [Metalink - Mozilla Wiki](https://wiki.mozilla.org/Metalink)
-* [Metalink - Wikipedia](https://en.wikipedia.org/wiki/Metalink)
-* [RFC 5854 - The Metalink Download Description Format](https://datatracker.ietf.org/doc/html/rfc5854)
-* [RFC 6249 - Metalink/HTTP: Mirrors and Hashes](https://www.rfc-editor.org/rfc/rfc6249.html)
-
-## Links
-
-* Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
-* Issues: [https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
-
-# Authors
-
-* [@sneak <sneak@sneak.berlin>](mailto:sneak@sneak.berlin)
-
-# License
-
-* [WTFPL](https://wtfpl.net)

TODO.md

@@ -1,122 +0,0 @@
-# TODO: mfer 1.0
-
-## Design Questions
-
-*sneak: please answer inline below each question. These are preserved for posterity.*
-
-### Format Design
-
-**1. Should `MFFileChecksum` be simplified?**
-Currently it's a separate message wrapping a single `bytes multiHash` field. Since multihash already self-describes the algorithm, `repeated bytes hashes` directly on `MFFilePath` would be simpler and reduce per-file protobuf overhead. Is the extra message layer intentional (e.g. planning to add per-hash metadata like `verified_at`)?
-
-> *answer:*
-
-**2. Should file permissions/mode be stored?**
-The format stores mtime/ctime but not Unix file permissions. For archival use (ExFAT, filesystem-independent checksums) this may not matter, but for software distribution or filesystem restoration it's a gap. Should we reserve a field now (e.g. `optional uint32 mode = 305`) even if we don't populate it yet?
-
-> *answer:*
-
-**3. Should `atime` be removed from the schema?**
-Access time is volatile, non-deterministic, and often disabled (`noatime`). Including it means two manifests of the same directory at different times will differ, which conflicts with the determinism goal. Remove it, or document it as "never set by default"?
-
-> *answer:*
-
-**4. What are the path normalization rules?**
-The proto has `string path` with no specification about: always forward-slash? Must be relative? No `..` components allowed? UTF-8 NFC vs NFD normalization (macOS vs Linux)? Max path length? This is a security issue (path traversal) and a cross-platform compatibility issue. What rules should the spec mandate?
-
-> *answer:*
-
-**5. Should we add a version byte after the magic?**
-Currently `ZNAVSRFG` is followed immediately by protobuf. Adding a version byte (`ZNAVSRFG\x01`) would allow future framing changes without requiring protobuf parsing to detect the version. `MFFileOuter.Version` serves this purpose but requires successful deserialization to read. Worth the extra byte?
-
-> *answer:*
-
-**6. Should we add a length-prefix after the magic?**
-Protobuf is not self-delimiting. If we ever want to concatenate manifests or append data after the protobuf, the current framing is insufficient. Add a varint or fixed-width length-prefix?
-
-> *answer:*
-
-### Signature Design
-
-**7. What does the outer SHA-256 hash cover — compressed or uncompressed data?**
-The review notes it currently hashes compressed data (good for verifying before decompression), but this should be explicitly documented. Which is the intended behavior?
-
-> *answer:*
-
-**8. Should `signatureString()` sign raw bytes instead of a hex-encoded string?**
-Currently the canonical string is `MAGIC-UUID-MULTIHASH` with hex encoding, which adds a transformation layer. Signing the raw `sha256` bytes (or compressed `innerMessage` directly) would be simpler. Keep the string format or switch to raw bytes?
-
-> *answer:*
-
-**9. Should we support detached signature files (`.mf.sig`)?**
-Embedded signatures are better for single-file distribution. Detached `.mf.sig` files follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are simpler for HTTP serving. Support both modes?
-
-> *answer:*
-
-**10. GPG vs pure-Go crypto for signatures?**
-Shelling out to `gpg` is fragile (may not be installed, version-dependent output). `github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could go Ed25519/signify (simpler, no key management). Which direction?
-
-> *answer:*
-
-### Implementation Design
-
-**11. Should manifests be deterministic by default?**
-This means: sort file entries by path, omit `createdAt` timestamp (or make it opt-in), no `atime`. Should determinism be the default, with a `--include-timestamps` flag to opt in?
-
-> *answer:*
-
-**12. Should we consolidate or keep both scanner/checker implementations?**
-There are two parallel implementations: `mfer/scanner.go` + `mfer/checker.go` (typed with `FileSize`, `RelFilePath`) and `internal/scanner/` + `internal/checker/` (raw `int64`, `string`). The `mfer/` versions are superior. Delete the `internal/` versions?
-
-> *answer:*
-
-**13. Should the `manifest` type be exported?**
-Currently unexported with exported constructors (`New`, `NewFromPaths`, etc.). Consumers can't declare `var m *mfer.manifest`. Export the type, or define an interface?
-
-> *answer:*
-
-**14. What should the Go module path be for 1.0?**
-Currently mixed between `sneak.berlin/go/mfer` and `git.eeqj.de/sneak/mfer`. Which is canonical?
-
-> *answer:*
-
----
-
-## Implementation Plan
-
-### Phase 1: Foundation (format correctness)
-
-- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate on `mfer/` package versions; update CLI code
-- [ ] Add deterministic file ordering — sort entries by path (lexicographic, byte-order) in `Builder.Build()`; add test asserting byte-identical output from two runs
-- [ ] Add decompression size limit — `io.LimitReader` in `deserializeInner()` with `m.pbOuter.Size` as bound
-- [ ] Fix `errors.Is` dead code in checker — replace with `os.IsNotExist(err)` or `errors.Is(err, fs.ErrNotExist)`
-- [ ] Fix `AddFile` to verify size — check `totalRead == size` after reading, return error on mismatch
-- [ ] Specify path invariants — add proto comments (UTF-8, forward-slash, relative, no `..`, no leading `/`); validate in `Builder.AddFile` and `Builder.AddFileWithHash`
-
-### Phase 2: CLI polish
-
-- [ ] Fix flag naming — all CLI flags use kebab-case as primary (`--include-dotfiles`, `--follow-symlinks`)
-- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or `url.JoinPath()` instead of string concatenation
-- [ ] Add progress rate-limiting to Checker — throttle to once per second, matching Scanner
-- [ ] Add `--deterministic` flag (or make it default) — omit `createdAt`, sort files
-
-### Phase 3: Robustness
-
-- [ ] Replace GPG subprocess with pure-Go crypto — `github.com/ProtonMail/go-crypto` or Ed25519/signify
-- [ ] Add timeout to any remaining subprocess calls
-- [ ] Add fuzzing tests for `NewManifestFromReader`
-- [ ] Add retry logic to fetch — exponential backoff for transient HTTP errors
-
-### Phase 4: Format finalization
-
-- [ ] Remove or deprecate `atime` from proto (pending design question answer)
-- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future file permissions
-- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version 1
-- [ ] Write format specification document — separate from README: magic, outer structure, compression, inner structure, path invariants, signature scheme, canonical serialization
-
-### Phase 5: Release prep
-
-- [ ] Finalize Go module path
-- [ ] Audit all error messages for consistency and helpfulness
-- [ ] Add `--version` output matching SemVer
-- [ ] Tag v1.0.0

go.mod

@@ -1,6 +1,6 @@
 module git.eeqj.de/sneak/mfer
 
-go 1.22
+go 1.17
 
 require (
 	github.com/apex/log v1.9.0
@@ -10,6 +10,7 @@ require (
 	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.23.6
 	google.golang.org/protobuf v1.28.1
+
 )
 
 require (

go.sum

@@ -37,6 +37,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
 github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=

internal/bork/error.go

@@ -2,14 +2,10 @@ package bork
 
 import (
 	"errors"
-	"fmt"
 )
 
 var (
 	ErrMissingMagic  = errors.New("missing magic bytes in file")
 	ErrFileTruncated = errors.New("file/stream is truncated abnormally")
+	ErrFileIntegrity = errors.New("file/stream checksum failure")
 )
-
-func Newf(format string, args ...interface{}) error {
-	return fmt.Errorf(format, args...)
-}

internal/log/log.go

@@ -23,11 +23,20 @@ func DisableStyling() {
 	pterm.Fatal.Prefix.Text = ""
 }
 
+var TestingTraces bool = false
+
 func Init() {
 	log.SetHandler(acli.Default)
 	log.SetLevel(log.InfoLevel)
 }
 
+func Tracef(format string, args ...interface{}) {
+	if !TestingTraces {
+		return
+	}
+	DebugReal(fmt.Sprintf(format, args...), 2)
+}
+
 func Debugf(format string, args ...interface{}) {
 	DebugReal(fmt.Sprintf(format, args...), 2)
 }
@@ -45,10 +54,22 @@ func DebugReal(arg string, cs int) {
 	log.Debug(tag + arg)
 }
 
+func TraceDump(args ...interface{}) {
+	if !TestingTraces {
+		return
+	}
+	DebugReal(spew.Sdump(args...), 2)
+}
+
 func Dump(args ...interface{}) {
 	DebugReal(spew.Sdump(args...), 2)
 }
 
+func EnableTestingLogging() {
+	TestingTraces = true
+	EnableDebugLogging()
+}
+
 func EnableDebugLogging() {
 	SetLevel(log.DebugLevel)
 }

mfer/deserialize.go

@@ -3,6 +3,7 @@ package mfer
 import (
 	"bytes"
 	"compress/gzip"
+	"crypto/sha256"
 	"errors"
 	"io"
 
@@ -11,12 +12,35 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+func (m *manifest) validateProtoInner() error {
+	i := m.pbInner
+	if i.Version != MFFile_VERSION_ONE {
+		return errors.New("unknown version") // FIXME move to bork
+	}
+
+	if len(i.Files) == 0 {
+		return errors.New("manifest without files") // FIXME move to bork
+	}
+
+	for _, mfp := range m.pbInner.Files {
+		// there is no way we should be doing validateProtoInner()
+		// outside of a load into a blank/empty/new *manifest
+		if m.files != nil {
+			return errors.New("shouldn't happen, internal error")
+		}
+		m.files = make([]*manifestFile, 0)
+		// we can skip error handling here thanks to the magic of protobuf
+		m.addFileLoadTime(mfp)
+	}
+	return nil
+}
+
 func (m *manifest) validateProtoOuter() error {
 	if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
-		return errors.New("unknown version")
+		return errors.New("unknown version") // FIXME move to bork
 	}
 	if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
-		return errors.New("unknown compression type")
+		return errors.New("unknown compression type") // FIXME move to bork
 	}
 
 	bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
@@ -35,12 +59,26 @@ func (m *manifest) validateProtoOuter() error {
 
 	isize := len(dat)
 	if int64(isize) != m.pbOuter.Size {
-		log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
+		log.Tracef("truncated data, got %d expected %d", isize, m.pbOuter.Size)
 		return bork.ErrFileTruncated
 	}
-	log.Debugf("inner data size is %d", isize)
+	log.Tracef("inner data size is %d", isize)
-	log.Dump(dat)
+	log.TraceDump(dat)
-	log.Dump(m.pbOuter.Sha256)
+
+	// FIXME validate Sha256
+	log.TraceDump(m.pbOuter.Sha256)
+
+	h := sha256.New()
+	h.Write(dat)
+	shaGot := h.Sum(nil)
+
+	log.TraceDump("got: ", shaGot)
+	log.TraceDump("expected: ", m.pbOuter.Sha256)
+
+	if !bytes.Equal(shaGot, m.pbOuter.Sha256) {
+		m.pbOuter.InnerMessage = nil // don't try to mess with it
+		return bork.ErrFileIntegrity
+	}
 	return nil
 }
 
@@ -54,7 +92,7 @@ func validateMagic(dat []byte) bool {
 	return bytes.Equal(got, expected)
 }
 
-func NewFromProto(input io.Reader) (*manifest, error) {
+func NewFromReader(input io.Reader) (*manifest, error) {
 	m := New()
 	dat, err := io.ReadAll(input)
 	if err != nil {
@@ -69,7 +107,7 @@ func NewFromProto(input io.Reader) (*manifest, error) {
 	bb := bytes.NewBuffer(dat[ml:])
 	dat = bb.Bytes()
 
-	log.Dump(dat)
+	log.TraceDump(dat)
 
 	// deserialize:
 	m.pbOuter = new(MFFileOuter)
@@ -84,6 +122,22 @@ func NewFromProto(input io.Reader) (*manifest, error) {
 		return nil, ve
 	}
 
-	// FIXME TODO deserialize inner
+	m.pbInner = new(MFFile)
+	err = proto.Unmarshal(m.pbOuter.InnerMessage, m.pbInner)
+	if err != nil {
+		return nil, err
+	}
+
+	log.TraceDump(m.pbInner)
+
+	ve = m.validateProtoInner()
+	if ve != nil {
+		return nil, ve
+	}
+
+	m.rescanInternal()
+
+	log.TraceDump(m)
+
 	return m, nil
 }

mfer/example_test.go

@@ -8,6 +8,10 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func init() {
+	log.EnableTestingLogging()
+}
+
 func TestAPIExample(t *testing.T) {
 	// read from filesystem
 	m, err := NewFromFS(&ManifestScanOptions{
@@ -24,7 +28,7 @@ func TestAPIExample(t *testing.T) {
 	m.WriteTo(&buf)
 
 	// show serialized
-	log.Dump(buf.Bytes())
+	log.TraceDump(buf.Bytes())
 
 	// do it again
 	var buf2 bytes.Buffer
@@ -34,9 +38,9 @@ func TestAPIExample(t *testing.T) {
 	assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
 
 	// deserialize
-	m2, err := NewFromProto(&buf)
+	m2, err := NewFromReader(&buf)
 	assert.Nil(t, err)
 	assert.NotNil(t, m2)
 
-	log.Dump(m2)
+	log.TraceDump(m2)
 }

mfer/manifest.go

@@ -17,6 +17,8 @@ import (
 type manifestFile struct {
 	path      string
 	info      fs.FileInfo
+	size      int64
+	protoFile *MFFilePath
 }
 
 func (m *manifestFile) String() string {
@@ -83,7 +85,7 @@ func New() *manifest {
 }
 
 func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) {
-	log.Dump(inputPaths)
+	log.TraceDump(inputPaths)
 	m := New()
 	m.scanOptions = options
 	for _, p := range inputPaths {
@@ -109,6 +111,10 @@ func (m *manifest) GetFileCount() int64 {
 	return int64(len(m.files))
 }
 
+func (m *manifest) rescanInternal() {
+	panic("unimplemented")
+}
+
 func (m *manifest) GetTotalFileSize() int64 {
 	return m.totalFileSize
 }
@@ -130,7 +136,17 @@ func pathIsHidden(p string) bool {
 	}
 }
 
-func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
+func (m *manifest) addFileLoadTime(in *MFFilePath) {
+	nf := &manifestFile{
+		path:      in.Path,
+		size:      in.Size,
+		protoFile: in, // FIXME get rid of this eventually
+	}
+	m.files = append(m.files, nf)
+	m.totalFileSize = m.totalFileSize + in.Size
+}
+
+func (m *manifest) addFileScanTime(p string, fi fs.FileInfo, sfsIndex int) error {
 	if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) {
 		return nil
 	}
@@ -163,7 +179,7 @@ func (m *manifest) Scan() error {
 			return errors.New("invalid source fs")
 		}
 		e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error {
-			return m.addFile(p, info, idx)
+			return m.addFileScanTime(p, info, idx)
 		})
 		if e != nil {
 			return e

mfer/mf.proto

@@ -9,7 +9,7 @@ message Timestamp {
 
 message MFFileOuter {
     enum Version {
-        VERSION_NONE = 0;
+        VERSION_NONE = 0; // must have a zero enum by law
         VERSION_ONE = 1; // only one for now
     }
 
@@ -17,7 +17,7 @@ message MFFileOuter {
     Version version = 101;
 
     enum CompressionType {
-        COMPRESSION_NONE = 0;
+        COMPRESSION_NONE = 0; // must have a zero enum by law
         COMPRESSION_GZIP = 1;
     }
 
@@ -33,12 +33,10 @@ message MFFileOuter {
     // think we might use gosignify instead of gpg:
     // github.com/frankbraun/gosignify
 
-    //detached signature, ascii or binary
+    //detached signatures, ascii or binary
-    optional bytes signature = 201;
+    repeated bytes signatures = 201;
-    //full GPG key id
+    //full GPG signing public keys, ascii or binary
-    optional bytes signer = 202;
+    repeated bytes signingPubKeys = 203;
-    //full GPG signing public key, ascii or binary
-    optional bytes signingPubKey = 203;
 }
 
 message MFFilePath {
@@ -58,13 +56,13 @@ message MFFilePath {
 
 message MFFileChecksum {
     // 1.0 golang implementation must write a multihash here
-    // it's ok to only ever use/verify sha256 multihash
+    // it's ok to only ever use/verify sha256 multihash i think?
     bytes multiHash = 1;
 }
 
 message MFFile {
     enum Version {
-        VERSION_NONE = 0;
+        VERSION_NONE = 0;  // must have a zero enum by law
         VERSION_ONE = 1; // only one for now
     }
     Version version = 100;

mfer/mfer_test.go

@@ -21,7 +21,7 @@ var (
 )
 
 func init() {
-	log.EnableDebugLogging()
+	log.EnableTestingLogging()
 
 	// create test files and directories
 	af.MkdirAll("/a/b/c", 0o755)
@@ -70,5 +70,5 @@ func TestManifestGenerationTwo(t *testing.T) {
 	var buf bytes.Buffer
 	err = m.WriteTo(&buf)
 	assert.Nil(t, err)
-	log.Dump(buf.Bytes())
+	log.TraceDump(buf.Bytes())
 }

mfer/serialize.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"time"
 
+	"git.eeqj.de/sneak/mfer/internal/log"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -59,6 +60,7 @@ func (m *manifest) generateOuter() error {
 
 	h := sha256.New()
 	h.Write(innerData)
+	sha256 := h.Sum(nil)
 
 	idc := new(bytes.Buffer)
 	gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
@@ -72,10 +74,12 @@ func (m *manifest) generateOuter() error {
 
 	gzw.Close()
 
+	log.Tracef("calculated sha256 (uncompressed): %x\n", sha256)
+
 	o := &MFFileOuter{
 		InnerMessage:    idc.Bytes(),
 		Size:            int64(len(innerData)),
-		Sha256:          h.Sum(nil),
+		Sha256:          sha256,
 		Version:         MFFileOuter_VERSION_ONE,
 		CompressionType: MFFileOuter_COMPRESSION_GZIP,
 	}