From f97402e7ab65683fec2e538c309739c046c6300d Mon Sep 17 00:00:00 2001 From: clawbot Date: Tue, 13 Oct 2026 18:39:15 +0000 Subject: [PATCH] Finish docs/FORMAT.md as a specification with a test vector (closes #84) docs/FORMAT.md now says a .mf is always a whole file, readers ignore fields they do not know, and paths are compared byte for byte with no Unicode normalization. Deterministic Serialization describes what the writer does: entry order, protobuf encoding, what each entry holds, zstd settings and how the UUID is chosen, and no longer claims two runs give the same bytes without --seed or with --include-timestamps. A signed test vector made once by the code, with a key used for nothing else, is checked in as mfer/testdata/test-vector.mf, shown in the document, and pinned by its SHA-256 and verified by TestFormatTestVector. Model: opus-5-5 --- docs/FORMAT.md | 199 ++++++++++++++++++++++++++++++++--- mfer/deserialize_test.go | 38 +++++++ mfer/testdata/test-vector.mf | Bin 0 -> 1056 bytes 3 files changed, 222 insertions(+), 15 deletions(-) create mode 100644 mfer/testdata/test-vector.mf diff --git a/docs/FORMAT.md b/docs/FORMAT.md index 6758f88..da6ac16 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -20,9 +20,14 @@ An `.mf` file consists of two parts, concatenated: 2. **Outer message**: a Protocol Buffers serialized `MFFileOuter` message There is no length prefix or version byte between the magic and the protobuf -message. The protobuf message extends to the end of the file. +message. The protobuf message extends to the end of the file, so a `.mf` is +always a whole file; a different framing, such as one with a length prefix, +would need a different magic. The format's version is the outer message's +`version` field. See [`mfer/mf.proto`](../mfer/mf.proto) for exact field numbers and types. +Readers ignore any field they do not know, in any message, so a later version +can add a field without breaking 1.0 readers. ## Outer Message (`MFFileOuter`) @@ -34,7 +39,7 @@ The outer message contains: | `compressionType` | 102 | enum | Compression of `innerMessage`; must be `COMPRESSION_ZSTD` (1) | | `size` | 103 | int64 | Uncompressed size of `innerMessage` (corruption detection) | | `sha256` | 104 | bytes | SHA-256 hash of the **compressed** `innerMessage` (corruption detection) | -| `uuid` | 105 | bytes | Random v4 UUID; must match the inner message UUID | +| `uuid` | 105 | bytes | 16-byte UUID; must match the inner message UUID | | `innerMessage` | 199 | bytes | Zstd-compressed serialized `MFFile` message | | `signature` | 201 | bytes (optional) | One OpenPGP signature over the canonical string, armored or binary | | `signer` | 202 | bytes (optional) | Fingerprint of the signing primary key, in upper-case hex | @@ -64,12 +69,12 @@ for the signature, the signing key and the other outer fields. After decompressing `innerMessage`, the result is a serialized `MFFile` (referred to as the manifest): -| Field | Number | Type | Description | -| ----------- | ------ | --------------------- | ------------------------------------- | -| `version` | 100 | enum | Must be `VERSION_ONE` (1) | -| `files` | 101 | repeated `MFFilePath` | List of files in the manifest | -| `uuid` | 102 | bytes | Random v4 UUID; must match outer UUID | -| `createdAt` | 201 | Timestamp (optional) | When the manifest was created | +| Field | Number | Type | Description | +| ----------- | ------ | --------------------- | ----------------------------------- | +| `version` | 100 | enum | Must be `VERSION_ONE` (1) | +| `files` | 101 | repeated `MFFilePath` | List of files in the manifest | +| `uuid` | 102 | bytes | 16-byte UUID; must match outer UUID | +| `createdAt` | 201 | Timestamp (optional) | When the manifest was created | ## File Entries (`MFFilePath`) @@ -116,6 +121,10 @@ Implementations must validate these invariants when reading and writing manifests. Paths that violate these rules must be rejected, and a reader must reject a manifest that lists a path more than once. +A path holds each name exactly as the file system gives it, with no Unicode +normalization, and readers compare paths byte for byte. So `é` stored as one +code point and `é` stored as `e` followed by a combining accent are two paths. + ## Hash Format (`MFFileChecksum`) Each checksum is a single `bytes multiHash` field containing a @@ -180,15 +189,175 @@ whether or not the key has expired too. ## Deterministic Serialization -By default, manifests are generated deterministically: +The reference implementation writes the same inner message for the same files, +UUID and options, unless the options include `--include-timestamps`. It +compresses the same inner message to the same bytes: -- File entries are sorted by `path` in **lexicographic byte order** -- `createdAt` is omitted unless explicitly requested -- `mode` is `0000` unless explicitly requested +- File entries are sorted by `path` in byte order, so `B.txt` comes before + `a.txt`. +- Each message is encoded by the Go protobuf library with deterministic + marshaling, which writes fields in field-number order and leaves out the + fields that are not set. A field not declared `optional` counts as not set + when it holds its default value, zero or empty. +- Each entry holds only `path`, `size`, `hashes` with exactly one hash (the + SHA-256 multihash of the file's content), `mtime` and `mode`. `mtime` is the + file's modification time as the file system gives it, to the nanosecond, and + `mode` is `0000` unless `gen` or `freshen` is given `--include-permissions`. +- `createdAt` is left out unless `gen` or `freshen` is given + `--include-timestamps`. Then it holds the time the run started building the + manifest, to the nanosecond, so it differs on every run. +- The inner message is compressed into one zstd frame, with a checksum of its + content, by the encoder of the Go library `github.com/klauspost/compress` at + its `SpeedBestCompression` level and otherwise its default settings. That + encoder gives the same bytes for the same input only within one version of the + library. Another zstd encoder gives other bytes for the same inner message, + and so another `sha256`. +- `gen` and `freshen` choose a new random version 4 UUID on every run, except + that `gen` given `--seed` takes the first 16 bytes of the SHA-256 of the seed + string instead. Those bytes are not a version 4 UUID; readers accept any 16 + bytes. -This ensures that two independent runs over the same directory tree produce -byte-identical `.mf` files (assuming file contents and metadata have not -changed). +So two runs of `gen` over an unchanged tree write byte-identical files only when +both are given the same `--seed`, neither is given `--include-timestamps`, and +neither signs the manifest: a signature records the time it was made, and the +OpenPGP library adds a random salt to each one. + +## Test Vector + +[`mfer/testdata/test-vector.mf`](../mfer/testdata/test-vector.mf) is a signed +manifest the reference implementation made once, with the UUID +`92197fd5-32bc-4090-8aa9-f86e5518b56c` and an Ed25519 key that the OpenPGP +library made for this file and nothing else; its secret key was not kept. +`TestFormatTestVector` in `mfer/deserialize_test.go` checks that the file's +SHA-256, its canonical string and `signer` are the ones shown here, and verifies +its signature. + +A reader should load the file and accept its signature. A writer given the same +files and UUID should give the same inner message; its compressed bytes, and so +its `sha256` and signature, depend on its zstd encoder (see Deterministic +Serialization). + +The manifest has no `createdAt` and lists two files, each with mode `0000` and +the modification time 2026-10-13 12:00:00.5 UTC (`seconds` 1791892800, `nanos` +500000000): + +| `path` | `size` | Content | +| --------------- | ------ | ----------------------- | +| `dir/world.txt` | 6 | `world` and a line feed | +| `hello.txt` | 6 | `hello` and a line feed | + +The inner message, 164 bytes before compression: + +``` +00000000 a0 06 01 aa 06 46 0a 0d 64 69 72 2f 77 6f 72 6c |.....F..dir/worl| +00000010 64 2e 74 78 74 10 06 1a 24 0a 22 12 20 e2 58 d2 |d.txt...$.". .X.| +00000020 48 fd a9 4c 63 75 36 07 f7 c4 49 4e e0 fc be 92 |H..Lcu6...IN....| +00000030 f1 a7 6b fd ac 79 5c 9d 84 10 1e b3 17 f2 12 0c |..k..y\.........| +00000040 08 c0 ba b8 d6 06 10 80 ca b5 ee 01 aa 06 42 0a |..............B.| +00000050 09 68 65 6c 6c 6f 2e 74 78 74 10 06 1a 24 0a 22 |.hello.txt...$."| +00000060 12 20 58 91 b5 b5 22 d5 df 08 6d 0f f0 b1 10 fb |. X..."...m.....| +00000070 d9 d2 1b b4 fc 71 63 af 34 d0 82 86 a2 e8 46 f6 |.....qc.4.....F.| +00000080 be 03 f2 12 0c 08 c0 ba b8 d6 06 10 80 ca b5 ee |................| +00000090 01 b2 06 10 92 19 7f d5 32 bc 40 90 8a a9 f8 6e |........2.@....n| +000000a0 55 18 b5 6c |U..l| +``` + +Its compressed form is hashed into `sha256`: + +``` +2353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352 +``` + +So the canonical string is: + +``` +ZNAVSRFG-92197fd532bc40908aa9f86e5518b56c-12202353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352 +``` + +`signer` is `5E962A37590D855B467D20294947A60319611E6F`, and `signature` holds: + +``` +-----BEGIN PGP SIGNATURE----- + +wqsEABYIAF0FgmrOejEJEElHpgMZYR5vNRQAAAAAABwAEHNhbHRAbm90YXRpb25z +Lm9wZW5wZ3Bqcy5vcmddxfykLLTqmTD7G/jHGzY+FiEEXpYqN1kNhVtGfSApSUem +AxlhHm8AAH0hAQDI6oGxvk1sGqIcu3Abvs7DM/7EalxkND614N75FCg+ywEApP6L +oIvrZUgEw8DUb9JeK7TChOi6PUAHUFY75Fp6hAE= +=nfX5 +-----END PGP SIGNATURE----- +``` + +The whole file, 1056 bytes, whose SHA-256 is +`5864187d671b2a55f8e015f858248f92c51956b549cf548b6bb54477c2407fd4`: + +``` +00000000 5a 4e 41 56 53 52 46 47 a8 06 01 b0 06 01 b8 06 |ZNAVSRFG........| +00000010 a4 01 c2 06 20 23 53 55 32 05 f0 36 a8 8c ef ee |.... #SU2..6....| +00000020 3d 0d 90 49 cc d8 e6 8b b2 40 11 a0 b4 62 b5 86 |=..I.....@...b..| +00000030 7f 1f cc 63 52 ca 06 10 92 19 7f d5 32 bc 40 90 |...cR.......2.@.| +00000040 8a a9 f8 6e 55 18 b5 6c ba 0c a1 01 28 b5 2f fd |...nU..l....(./.| +00000050 04 00 a5 04 00 94 08 a0 06 01 aa 06 46 0a 0d 64 |............F..d| +00000060 69 72 2f 77 6f 72 6c 64 2e 74 78 74 10 06 1a 24 |ir/world.txt...$| +00000070 0a 22 12 20 e2 58 d2 48 fd a9 4c 63 75 36 07 f7 |.". .X.H..Lcu6..| +00000080 c4 49 4e e0 fc be 92 f1 a7 6b fd ac 79 5c 9d 84 |.IN......k..y\..| +00000090 10 1e b3 17 f2 12 0c 08 c0 ba b8 d6 06 10 80 ca |................| +000000a0 b5 ee 01 aa 06 42 0a 09 68 65 6c 6c 6f 58 91 b5 |.....B..helloX..| +000000b0 b5 22 d5 df 08 6d 0f f0 b1 10 fb d9 d2 1b b4 fc |."...m..........| +000000c0 71 63 af 34 d0 82 86 a2 e8 46 f6 be 03 b2 06 10 |qc.4.....F......| +000000d0 92 19 7f d5 32 bc 40 90 8a a9 f8 6e 55 18 b5 6c |....2.@....nU..l| +000000e0 02 10 06 40 ba b7 10 ce 0c f8 c2 24 13 ca 0c ac |...@.......$....| +000000f0 02 2d 2d 2d 2d 2d 42 45 47 49 4e 20 50 47 50 20 |.-----BEGIN PGP | +00000100 53 49 47 4e 41 54 55 52 45 2d 2d 2d 2d 2d 0a 0a |SIGNATURE-----..| +00000110 77 71 73 45 41 42 59 49 41 46 30 46 67 6d 72 4f |wqsEABYIAF0FgmrO| +00000120 65 6a 45 4a 45 45 6c 48 70 67 4d 5a 59 52 35 76 |ejEJEElHpgMZYR5v| +00000130 4e 52 51 41 41 41 41 41 41 42 77 41 45 48 4e 68 |NRQAAAAAABwAEHNh| +00000140 62 48 52 41 62 6d 39 30 59 58 52 70 62 32 35 7a |bHRAbm90YXRpb25z| +00000150 0a 4c 6d 39 77 5a 57 35 77 5a 33 42 71 63 79 35 |.Lm9wZW5wZ3Bqcy5| +00000160 76 63 6d 64 64 78 66 79 6b 4c 4c 54 71 6d 54 44 |vcmddxfykLLTqmTD| +00000170 37 47 2f 6a 48 47 7a 59 2b 46 69 45 45 58 70 59 |7G/jHGzY+FiEEXpY| +00000180 71 4e 31 6b 4e 68 56 74 47 66 53 41 70 53 55 65 |qN1kNhVtGfSApSUe| +00000190 6d 0a 41 78 6c 68 48 6d 38 41 41 48 30 68 41 51 |m.AxlhHm8AAH0hAQ| +000001a0 44 49 36 6f 47 78 76 6b 31 73 47 71 49 63 75 33 |DI6oGxvk1sGqIcu3| +000001b0 41 62 76 73 37 44 4d 2f 37 45 61 6c 78 6b 4e 44 |Abvs7DM/7EalxkND| +000001c0 36 31 34 4e 37 35 46 43 67 2b 79 77 45 41 70 50 |614N75FCg+ywEApP| +000001d0 36 4c 0a 6f 49 76 72 5a 55 67 45 77 38 44 55 62 |6L.oIvrZUgEw8DUb| +000001e0 39 4a 65 4b 37 54 43 68 4f 69 36 50 55 41 48 55 |9JeK7TChOi6PUAHU| +000001f0 46 59 37 35 46 70 36 68 41 45 3d 0a 3d 6e 66 58 |FY75Fp6hAE=.=nfX| +00000200 35 0a 2d 2d 2d 2d 2d 45 4e 44 20 50 47 50 20 53 |5.-----END PGP S| +00000210 49 47 4e 41 54 55 52 45 2d 2d 2d 2d 2d d2 0c 28 |IGNATURE-----..(| +00000220 35 45 39 36 32 41 33 37 35 39 30 44 38 35 35 42 |5E962A37590D855B| +00000230 34 36 37 44 32 30 32 39 34 39 34 37 41 36 30 33 |467D20294947A603| +00000240 31 39 36 31 31 45 36 46 da 0c d4 03 2d 2d 2d 2d |19611E6F....----| +00000250 2d 42 45 47 49 4e 20 50 47 50 20 50 55 42 4c 49 |-BEGIN PGP PUBLI| +00000260 43 20 4b 45 59 20 42 4c 4f 43 4b 2d 2d 2d 2d 2d |C KEY BLOCK-----| +00000270 0a 0a 78 6a 4d 45 61 73 31 30 67 42 59 4a 4b 77 |..xjMEas10gBYJKw| +00000280 59 42 42 41 48 61 52 77 38 42 41 51 64 41 70 71 |YBBAHaRw8BAQdApq| +00000290 33 32 53 79 4f 72 4e 75 76 31 4d 71 53 55 66 45 |32SyOrNuv1MqSUfE| +000002a0 44 67 74 42 7a 6e 4b 59 32 39 38 2f 30 47 46 55 |DgtBznKY298/0GFU| +000002b0 76 6b 0a 6c 6f 42 71 35 69 6e 4e 4b 47 31 6d 5a |vk.loBq5inNKG1mZ| +000002c0 58 49 67 64 47 56 7a 64 43 42 32 5a 57 4e 30 62 |XIgdGVzdCB2ZWN0b| +000002d0 33 49 67 50 48 52 6c 63 33 51 74 64 6d 56 6a 64 |3IgPHRlc3QtdmVjd| +000002e0 47 39 79 51 47 31 6d 5a 58 49 75 64 47 56 7a 64 |G9yQG1mZXIudGVzd| +000002f0 44 37 43 0a 76 51 51 54 46 67 67 41 62 77 57 43 |D7C.vQQTFggAbwWC| +00000300 61 73 31 30 67 41 49 4c 42 77 6b 51 53 55 65 6d |as10gAILBwkQSUem| +00000310 41 78 6c 68 48 6d 38 31 46 41 41 41 41 41 41 41 |AxlhHm81FAAAAAAA| +00000320 48 41 41 51 63 32 46 73 64 45 42 75 62 33 52 68 |HAAQc2FsdEBub3Rh| +00000330 64 47 6c 76 0a 62 6e 4d 75 62 33 42 6c 62 6e 42 |dGlv.bnMub3BlbnB| +00000340 6e 63 47 70 7a 4c 6d 39 79 5a 30 79 62 6c 73 34 |ncGpzLm9yZ0ybls4| +00000350 71 53 43 6d 77 4a 64 45 49 45 6d 4c 72 54 55 45 |qSCmwJdEIEmLrTUE| +00000360 43 46 51 67 43 46 67 41 43 47 51 45 43 6d 77 4d |CFQgCFgACGQECmwM| +00000370 43 48 67 45 57 0a 49 51 52 65 6c 69 6f 33 57 51 |CHgEW.IQRelio3WQ| +00000380 32 46 57 30 5a 39 49 43 6c 4a 52 36 59 44 47 57 |2FW0Z9IClJR6YDGW| +00000390 45 65 62 77 41 41 31 67 73 41 2f 32 61 2f 50 58 |EebwAA1gsA/2a/PX| +000003a0 68 6c 44 49 4b 78 67 32 79 4e 30 4c 58 39 48 68 |hlDIKxg2yN0LX9Hh| +000003b0 49 44 43 54 7a 69 0a 49 32 43 53 71 44 74 2b 34 |IDCTzi.I2CSqDt+4| +000003c0 62 2f 73 7a 6d 59 64 41 51 43 66 2b 63 6a 6b 4f |b/szmYdAQCf+cjkO| +000003d0 71 6c 34 4c 47 5a 41 43 6c 2f 73 2f 2f 6d 31 71 |ql4LGZACl/s//m1q| +000003e0 62 74 6e 2f 4a 31 46 45 53 52 56 73 4e 68 65 2f |btn/J1FESRVsNhe/| +000003f0 76 6d 69 41 67 3d 3d 0a 3d 56 6b 4f 33 0a 2d 2d |vmiAg==.=VkO3.--| +00000400 2d 2d 2d 45 4e 44 20 50 47 50 20 50 55 42 4c 49 |---END PGP PUBLI| +00000410 43 20 4b 45 59 20 42 4c 4f 43 4b 2d 2d 2d 2d 2d |C KEY BLOCK-----| +``` ## MIME Type diff --git a/mfer/deserialize_test.go b/mfer/deserialize_test.go index a8b53d6..dd54d12 100644 --- a/mfer/deserialize_test.go +++ b/mfer/deserialize_test.go @@ -3,6 +3,9 @@ package mfer import ( "bytes" + "crypto/sha256" + "encoding/hex" + "os" "testing" "uuid" @@ -52,3 +55,38 @@ func TestReadAtMost(t *testing.T) { "file exceeds maximum allowed size of 65536 bytes") assert.Equal(t, maxSize-1, input.Len(), "bytes left unread") } + +// TestFormatTestVector checks the test vector that docs/FORMAT.md shows: the +// SHA-256 of the whole file, so that any change to the file fails, then loads +// it, verifies its signature, and checks the canonical string and the signer +// the document gives for it. +func TestFormatTestVector(t *testing.T) { + t.Parallel() + + const ( + path = "testdata/test-vector.mf" + fileSHA256 = "5864187d671b2a55f8e015f858248f92c51956b549cf548b6bb54477c2407fd4" + signer = "5E962A37590D855B467D20294947A60319611E6F" + ) + + data, err := os.ReadFile(path) + require.NoError(t, err) + + sum := sha256.Sum256(data) + require.Equal(t, fileSHA256, hex.EncodeToString(sum[:])) + + m, err := NewManifestFromFile(&ManifestFromFileOptions{Path: path}) + require.NoError(t, err) + + canonicalString, err := m.signatureString() + require.NoError(t, err) + assert.Equal(t, "ZNAVSRFG-92197fd532bc40908aa9f86e5518b56c-"+ + "12202353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352", + canonicalString) + + keyFingerprint, err := verifySignature([]byte(canonicalString), + m.pbOuter.GetSignature(), m.pbOuter.GetSigningPubKey()) + require.NoError(t, err) + assert.Equal(t, signer, keyFingerprint) + assert.Equal(t, signer, string(m.pbOuter.GetSigner())) +} diff --git a/mfer/testdata/test-vector.mf b/mfer/testdata/test-vector.mf new file mode 100644 index 0000000000000000000000000000000000000000..d3ddbfe41cc05dea2fa7d31825c4952157a2fbff GIT binary patch literal 1056 zcmZvZU8vh+7{}Lb)??_f&I`Xl%iP7_p!sN%o?IL_Z<043ZJIV|(>B4Ilcr7DCT-J? zr0vZ_WEbki9Nr336gL^DbH#CxA!ApEG34Mxy-~anujceZ2kwAfHxO~p-SfPE9-jC2 zGZj!RYrJqS9{V&NyBz-@b~T=Qv8)r1+@{XG_1mq@Bd4Y7-~ICD#jPjKUa}UaPQGxx zq3y<#XP!BEllXG$^c&~z^z^3}-cI7Z*og(YfB3+MhY!57b~YaSIL;$S?Di0i`a{ot zc|08_iu8&Kd&6R z`2V{OCgWQ>UnFlN?p!_oawYI9Tvk9kqA)ks`8I_0fe|QpS8=Ojp;ZQhvflI&F!fx~-vB_wT%d9i)fc8= z7as|M)R?4!6^^o80nNf~Z`xHj3ePB6l4qUuSp>nLM9D~B3Wuidz-WWhEjriCXDh5* zY*Qs2h&pd9(*w!{a1+_=wdy3Y`Z!d$hv3*toFE}h5g?rys%MlK&23~yAt>uD$T?$X-pd;Vy@6r^uZLa4>oWn_?kRZz_szQG*g|z~vkYO@ z6wFFXPD#|fMp_-)ezju@^h~{3n8lGs_!gf->tzIU7pQmxmP{vb^fB zyaQN4h0B)$D>|@-NUGNK+WmA*CHNX<(h}?CG|J$F8f;n-0Jt*(D6x%}>aNF0`Kd$9 z6ilwuqAPK1W!^?4f-MK!cs*mGqq%R`KxJF&jZU{1co|tR0qdb76!r1I8u!o~&cm`+ Y9Vu=T4gEH7Hka