Finish docs/FORMAT.md as a specification with a test vector (closes #84)

docs/FORMAT.md now says a .mf is always a whole file, readers ignore
fields they do not know, and paths are compared byte for byte with no
Unicode normalization. Deterministic Serialization describes what the
writer does: entry order, protobuf encoding, what each entry holds, zstd
settings and how the UUID is chosen, and no longer claims two runs give
the same bytes without --seed or with --include-timestamps. A signed
test vector made once by the code, with a key used for nothing else, is
checked in as mfer/testdata/test-vector.mf, shown in the document, and
pinned by its SHA-256 and verified by TestFormatTestVector.

Model: opus-5-5

docs/FORMAT.md

@@ -190,7 +190,8 @@ whether or not the key has expired too.
 ## Deterministic Serialization
 
 The reference implementation writes the same inner message for the same files,
-UUID and options, and compresses the same inner message to the same bytes:
+UUID and options, unless the options include `--include-timestamps`. It
+compresses the same inner message to the same bytes:
 
 - File entries are sorted by `path` in byte order, so `B.txt` comes before
   `a.txt`.
@@ -198,10 +199,13 @@ UUID and options, and compresses the same inner message to the same bytes:
   marshaling, which writes fields in field-number order and leaves out the
   fields that are not set. A field not declared `optional` counts as not set
   when it holds its default value, zero or empty.
-- Each entry's `mtime` is the file's modification time as the file system gives
-  it, to the nanosecond. `createdAt` is left out unless `gen` or `freshen` is
-  given `--include-timestamps`, and `mode` is `0000` unless it is given
-  `--include-permissions`.
+- Each entry holds only `path`, `size`, `hashes` with exactly one hash (the
+  SHA-256 multihash of the file's content), `mtime` and `mode`. `mtime` is the
+  file's modification time as the file system gives it, to the nanosecond, and
+  `mode` is `0000` unless `gen` or `freshen` is given `--include-permissions`.
+- `createdAt` is left out unless `gen` or `freshen` is given
+  `--include-timestamps`. Then it holds the time the run started building the
+  manifest, to the nanosecond, so it differs on every run.
 - The inner message is compressed into one zstd frame, with a checksum of its
   content, by the encoder of the Go library `github.com/klauspost/compress` at
   its `SpeedBestCompression` level and otherwise its default settings. That
@@ -214,9 +218,9 @@ UUID and options, and compresses the same inner message to the same bytes:
   bytes.
 
 So two runs of `gen` over an unchanged tree write byte-identical files only when
-both are given the same `--seed` and neither signs the manifest: a signature
-records the time it was made, and the OpenPGP library adds a random salt to each
-one.
+both are given the same `--seed`, neither is given `--include-timestamps`, and
+neither signs the manifest: a signature records the time it was made, and the
+OpenPGP library adds a random salt to each one.
 
 ## Test Vector
 
@@ -224,8 +228,9 @@ one.
 manifest the reference implementation made once, with the UUID
 `92197fd5-32bc-4090-8aa9-f86e5518b56c` and an Ed25519 key that the OpenPGP
 library made for this file and nothing else; its secret key was not kept.
-`TestFormatTestVector` in `mfer/deserialize_test.go` loads the file, verifies
-its signature and checks the signed string and `signer` shown here.
+`TestFormatTestVector` in `mfer/deserialize_test.go` checks that the file's
+SHA-256, its canonical string and `signer` are the ones shown here, and verifies
+its signature.
 
 A reader should load the file and accept its signature. A writer given the same
 files and UUID should give the same inner message; its compressed bytes, and so
@@ -263,7 +268,7 @@ Its compressed form is hashed into `sha256`:
 2353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352
 ```
 
-So the signed string is:
+So the canonical string is:
 
 ```
 ZNAVSRFG-92197fd532bc40908aa9f86e5518b56c-12202353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352
@@ -282,7 +287,8 @@ oIvrZUgEw8DUb9JeK7TChOi6PUAHUFY75Fp6hAE=
 -----END PGP SIGNATURE-----
 ```
 
-The whole file, 1056 bytes:
+The whole file, 1056 bytes, whose SHA-256 is
+`5864187d671b2a55f8e015f858248f92c51956b549cf548b6bb54477c2407fd4`:
 
 ```
 00000000  5a 4e 41 56 53 52 46 47  a8 06 01 b0 06 01 b8 06  |ZNAVSRFG........|

mfer/deserialize_test.go

@@ -3,6 +3,9 @@ package mfer
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"os"
 	"testing"
 	"uuid"
 
@@ -53,26 +56,35 @@ func TestReadAtMost(t *testing.T) {
 	assert.Equal(t, maxSize-1, input.Len(), "bytes left unread")
 }
 
-// TestFormatTestVector loads the test vector that docs/FORMAT.md shows,
-// verifies its signature, and checks the signed string and the signer the
-// document gives for it.
+// TestFormatTestVector checks the test vector that docs/FORMAT.md shows: the
+// SHA-256 of the whole file, so that any change to the file fails, then loads
+// it, verifies its signature, and checks the canonical string and the signer
+// the document gives for it.
 func TestFormatTestVector(t *testing.T) {
 	t.Parallel()
 
-	const signer = "5E962A37590D855B467D20294947A60319611E6F"
+	const (
+		path       = "testdata/test-vector.mf"
+		fileSHA256 = "5864187d671b2a55f8e015f858248f92c51956b549cf548b6bb54477c2407fd4"
+		signer     = "5E962A37590D855B467D20294947A60319611E6F"
+	)
 
-	m, err := NewManifestFromFile(&ManifestFromFileOptions{
-		Path: "testdata/test-vector.mf",
-	})
+	data, err := os.ReadFile(path)
 	require.NoError(t, err)
 
-	signedString, err := m.signatureString()
+	sum := sha256.Sum256(data)
+	require.Equal(t, fileSHA256, hex.EncodeToString(sum[:]))
+
+	m, err := NewManifestFromFile(&ManifestFromFileOptions{Path: path})
+	require.NoError(t, err)
+
+	canonicalString, err := m.signatureString()
 	require.NoError(t, err)
 	assert.Equal(t, "ZNAVSRFG-92197fd532bc40908aa9f86e5518b56c-"+
 		"12202353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352",
-		signedString)
+		canonicalString)
 
-	keyFingerprint, err := verifySignature([]byte(signedString),
+	keyFingerprint, err := verifySignature([]byte(canonicalString),
 		m.pbOuter.GetSignature(), m.pbOuter.GetSigningPubKey())
 	require.NoError(t, err)
 	assert.Equal(t, signer, keyFingerprint)