Finish docs/FORMAT.md as a specification with a test vector (closes #84)

docs/FORMAT.md now says a .mf is always a whole file, readers ignore
fields they do not know, and paths are compared byte for byte with no
Unicode normalization. Deterministic Serialization describes what the
writer does: entry order, protobuf encoding, zstd settings and how the
UUID is chosen, and no longer claims two runs give the same bytes
without --seed. A signed test vector made once by the code, with a key
used for nothing else, is checked in as mfer/testdata/test-vector.mf,
shown in the document, and loaded and verified by TestFormatTestVector.

Model: opus-5-5

docs/FORMAT.md

@@ -20,9 +20,14 @@ An `.mf` file consists of two parts, concatenated:
 2. **Outer message**: a Protocol Buffers serialized `MFFileOuter` message
 
 There is no length prefix or version byte between the magic and the protobuf
-message. The protobuf message extends to the end of the file.
+message. The protobuf message extends to the end of the file, so a `.mf` is
+always a whole file; a different framing, such as one with a length prefix,
+would need a different magic. The format's version is the outer message's
+`version` field.
 
 See [`mfer/mf.proto`](../mfer/mf.proto) for exact field numbers and types.
+Readers ignore any field they do not know, in any message, so a later version
+can add a field without breaking 1.0 readers.
 
 ## Outer Message (`MFFileOuter`)
 
@@ -34,7 +39,7 @@ The outer message contains:
 | `compressionType` | 102    | enum             | Compression of `innerMessage`; must be `COMPRESSION_ZSTD` (1)            |
 | `size`            | 103    | int64            | Uncompressed size of `innerMessage` (corruption detection)               |
 | `sha256`          | 104    | bytes            | SHA-256 hash of the **compressed** `innerMessage` (corruption detection) |
-| `uuid`            | 105    | bytes            | Random v4 UUID; must match the inner message UUID                        |
+| `uuid`            | 105    | bytes            | 16-byte UUID; must match the inner message UUID                          |
 | `innerMessage`    | 199    | bytes            | Zstd-compressed serialized `MFFile` message                              |
 | `signature`       | 201    | bytes (optional) | One OpenPGP signature over the canonical string, armored or binary       |
 | `signer`          | 202    | bytes (optional) | Fingerprint of the signing primary key, in upper-case hex                |
@@ -64,26 +69,26 @@ for the signature, the signing key and the other outer fields.
 After decompressing `innerMessage`, the result is a serialized `MFFile`
 (referred to as the manifest):
 
-| Field       | Number | Type                  | Description                           |
+| Field       | Number | Type                  | Description                         |
-| ----------- | ------ | --------------------- | ------------------------------------- |
+| ----------- | ------ | --------------------- | ----------------------------------- |
-| `version`   | 100    | enum                  | Must be `VERSION_ONE` (1)             |
+| `version`   | 100    | enum                  | Must be `VERSION_ONE` (1)           |
-| `files`     | 101    | repeated `MFFilePath` | List of files in the manifest         |
+| `files`     | 101    | repeated `MFFilePath` | List of files in the manifest       |
-| `uuid`      | 102    | bytes                 | Random v4 UUID; must match outer UUID |
+| `uuid`      | 102    | bytes                 | 16-byte UUID; must match outer UUID |
-| `createdAt` | 201    | Timestamp (optional)  | When the manifest was created         |
+| `createdAt` | 201    | Timestamp (optional)  | When the manifest was created       |
 
 ## File Entries (`MFFilePath`)
 
 Each file entry contains:
 
-| Field      | Number | Type                      | Description                         |
+| Field      | Number | Type                      | Description                                                   |
-| ---------- | ------ | ------------------------- | ----------------------------------- |
+| ---------- | ------ | ------------------------- | ------------------------------------------------------------- |
-| `path`     | 1      | string                    | Relative file path (see Path Rules) |
+| `path`     | 1      | string                    | Relative file path (see Path Rules)                           |
-| `size`     | 2      | int64                     | File size in bytes                  |
+| `size`     | 2      | int64                     | File size in bytes                                            |
-| `hashes`   | 3      | repeated `MFFileChecksum` | At least one hash required          |
+| `hashes`   | 3      | repeated `MFFileChecksum` | At least one hash required; readers reject an entry with none |
-| `mimeType` | 301    | string (optional)         | MIME type                           |
+| `mimeType` | 301    | string (optional)         | MIME type                                                     |
-| `mtime`    | 302    | Timestamp (optional)      | Modification time                   |
+| `mtime`    | 302    | Timestamp (optional)      | Modification time                                             |
-| `ctime`    | 303    | Timestamp (optional)      | Change time (inode metadata change) |
+| `ctime`    | 303    | Timestamp (optional)      | Change time (inode metadata change)                           |
-| `mode`     | 304    | uint32                    | Permission bits (see File Mode)     |
+| `mode`     | 304    | uint32                    | Permission bits (see File Mode)                               |
 
 ## File Mode
 
@@ -116,6 +121,10 @@ Implementations must validate these invariants when reading and writing
 manifests. Paths that violate these rules must be rejected, and a reader must
 reject a manifest that lists a path more than once.
 
+A path holds each name exactly as the file system gives it, with no Unicode
+normalization, and readers compare paths byte for byte. So `é` stored as one
+code point and `é` stored as `e` followed by a combining accent are two paths.
+
 ## Hash Format (`MFFileChecksum`)
 
 Each checksum is a single `bytes multiHash` field containing a
@@ -174,17 +183,175 @@ also refuses a manifest whose `signingPubKey` holds a DSA key or subkey, or any
 secret key or subkey, since checking the self-signatures of a DSA key or the
 numbers of a secret key can take hours when those numbers are very large.
 
+A signing key that has expired does not make the signature bad, because a
+manifest outlives its key. A signature past its own expiry time is not good,
+whether or not the key has expired too.
+
 ## Deterministic Serialization
 
-By default, manifests are generated deterministically:
+The reference implementation writes the same inner message for the same files,
+UUID and options, and compresses the same inner message to the same bytes:
 
-- File entries are sorted by `path` in **lexicographic byte order**
+- File entries are sorted by `path` in byte order, so `B.txt` comes before
-- `createdAt` is omitted unless explicitly requested
+  `a.txt`.
-- `mode` is `0000` unless explicitly requested
+- Each message is encoded by the Go protobuf library with deterministic
+  marshaling, which writes fields in field-number order and leaves out the
+  fields that are not set. A field not declared `optional` counts as not set
+  when it holds its default value, zero or empty.
+- Each entry's `mtime` is the file's modification time as the file system gives
+  it, to the nanosecond. `createdAt` is left out unless `gen` or `freshen` is
+  given `--include-timestamps`, and `mode` is `0000` unless it is given
+  `--include-permissions`.
+- The inner message is compressed into one zstd frame, with a checksum of its
+  content, by the encoder of the Go library `github.com/klauspost/compress` at
+  its `SpeedBestCompression` level and otherwise its default settings. That
+  encoder gives the same bytes for the same input only within one version of the
+  library. Another zstd encoder gives other bytes for the same inner message,
+  and so another `sha256`.
+- `gen` and `freshen` choose a new random version 4 UUID on every run, except
+  that `gen` given `--seed` takes the first 16 bytes of the SHA-256 of the seed
+  string instead. Those bytes are not a version 4 UUID; readers accept any 16
+  bytes.
 
-This ensures that two independent runs over the same directory tree produce
+So two runs of `gen` over an unchanged tree write byte-identical files only when
-byte-identical `.mf` files (assuming file contents and metadata have not
+both are given the same `--seed` and neither signs the manifest: a signature
-changed).
+records the time it was made, and the OpenPGP library adds a random salt to each
+one.
+
+## Test Vector
+
+[`mfer/testdata/test-vector.mf`](../mfer/testdata/test-vector.mf) is a signed
+manifest the reference implementation made once, with the UUID
+`92197fd5-32bc-4090-8aa9-f86e5518b56c` and an Ed25519 key that the OpenPGP
+library made for this file and nothing else; its secret key was not kept.
+`TestFormatTestVector` in `mfer/deserialize_test.go` loads the file, verifies
+its signature and checks the signed string and `signer` shown here.
+
+A reader should load the file and accept its signature. A writer given the same
+files and UUID should give the same inner message; its compressed bytes, and so
+its `sha256` and signature, depend on its zstd encoder (see Deterministic
+Serialization).
+
+The manifest has no `createdAt` and lists two files, each with mode `0000` and
+the modification time 2026-10-13 12:00:00.5 UTC (`seconds` 1791892800, `nanos`
+500000000):
+
+| `path`          | `size` | Content                 |
+| --------------- | ------ | ----------------------- |
+| `dir/world.txt` | 6      | `world` and a line feed |
+| `hello.txt`     | 6      | `hello` and a line feed |
+
+The inner message, 164 bytes before compression:
+
+```
+00000000  a0 06 01 aa 06 46 0a 0d  64 69 72 2f 77 6f 72 6c  |.....F..dir/worl|
+00000010  64 2e 74 78 74 10 06 1a  24 0a 22 12 20 e2 58 d2  |d.txt...$.". .X.|
+00000020  48 fd a9 4c 63 75 36 07  f7 c4 49 4e e0 fc be 92  |H..Lcu6...IN....|
+00000030  f1 a7 6b fd ac 79 5c 9d  84 10 1e b3 17 f2 12 0c  |..k..y\.........|
+00000040  08 c0 ba b8 d6 06 10 80  ca b5 ee 01 aa 06 42 0a  |..............B.|
+00000050  09 68 65 6c 6c 6f 2e 74  78 74 10 06 1a 24 0a 22  |.hello.txt...$."|
+00000060  12 20 58 91 b5 b5 22 d5  df 08 6d 0f f0 b1 10 fb  |. X..."...m.....|
+00000070  d9 d2 1b b4 fc 71 63 af  34 d0 82 86 a2 e8 46 f6  |.....qc.4.....F.|
+00000080  be 03 f2 12 0c 08 c0 ba  b8 d6 06 10 80 ca b5 ee  |................|
+00000090  01 b2 06 10 92 19 7f d5  32 bc 40 90 8a a9 f8 6e  |........2.@....n|
+000000a0  55 18 b5 6c                                       |U..l|
+```
+
+Its compressed form is hashed into `sha256`:
+
+```
+2353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352
+```
+
+So the signed string is:
+
+```
+ZNAVSRFG-92197fd532bc40908aa9f86e5518b56c-12202353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352
+```
+
+`signer` is `5E962A37590D855B467D20294947A60319611E6F`, and `signature` holds:
+
+```
+-----BEGIN PGP SIGNATURE-----
+
+wqsEABYIAF0FgmrOejEJEElHpgMZYR5vNRQAAAAAABwAEHNhbHRAbm90YXRpb25z
+Lm9wZW5wZ3Bqcy5vcmddxfykLLTqmTD7G/jHGzY+FiEEXpYqN1kNhVtGfSApSUem
+AxlhHm8AAH0hAQDI6oGxvk1sGqIcu3Abvs7DM/7EalxkND614N75FCg+ywEApP6L
+oIvrZUgEw8DUb9JeK7TChOi6PUAHUFY75Fp6hAE=
+=nfX5
+-----END PGP SIGNATURE-----
+```
+
+The whole file, 1056 bytes:
+
+```
+00000000  5a 4e 41 56 53 52 46 47  a8 06 01 b0 06 01 b8 06  |ZNAVSRFG........|
+00000010  a4 01 c2 06 20 23 53 55  32 05 f0 36 a8 8c ef ee  |.... #SU2..6....|
+00000020  3d 0d 90 49 cc d8 e6 8b  b2 40 11 a0 b4 62 b5 86  |=..I.....@...b..|
+00000030  7f 1f cc 63 52 ca 06 10  92 19 7f d5 32 bc 40 90  |...cR.......2.@.|
+00000040  8a a9 f8 6e 55 18 b5 6c  ba 0c a1 01 28 b5 2f fd  |...nU..l....(./.|
+00000050  04 00 a5 04 00 94 08 a0  06 01 aa 06 46 0a 0d 64  |............F..d|
+00000060  69 72 2f 77 6f 72 6c 64  2e 74 78 74 10 06 1a 24  |ir/world.txt...$|
+00000070  0a 22 12 20 e2 58 d2 48  fd a9 4c 63 75 36 07 f7  |.". .X.H..Lcu6..|
+00000080  c4 49 4e e0 fc be 92 f1  a7 6b fd ac 79 5c 9d 84  |.IN......k..y\..|
+00000090  10 1e b3 17 f2 12 0c 08  c0 ba b8 d6 06 10 80 ca  |................|
+000000a0  b5 ee 01 aa 06 42 0a 09  68 65 6c 6c 6f 58 91 b5  |.....B..helloX..|
+000000b0  b5 22 d5 df 08 6d 0f f0  b1 10 fb d9 d2 1b b4 fc  |."...m..........|
+000000c0  71 63 af 34 d0 82 86 a2  e8 46 f6 be 03 b2 06 10  |qc.4.....F......|
+000000d0  92 19 7f d5 32 bc 40 90  8a a9 f8 6e 55 18 b5 6c  |....2.@....nU..l|
+000000e0  02 10 06 40 ba b7 10 ce  0c f8 c2 24 13 ca 0c ac  |...@.......$....|
+000000f0  02 2d 2d 2d 2d 2d 42 45  47 49 4e 20 50 47 50 20  |.-----BEGIN PGP |
+00000100  53 49 47 4e 41 54 55 52  45 2d 2d 2d 2d 2d 0a 0a  |SIGNATURE-----..|
+00000110  77 71 73 45 41 42 59 49  41 46 30 46 67 6d 72 4f  |wqsEABYIAF0FgmrO|
+00000120  65 6a 45 4a 45 45 6c 48  70 67 4d 5a 59 52 35 76  |ejEJEElHpgMZYR5v|
+00000130  4e 52 51 41 41 41 41 41  41 42 77 41 45 48 4e 68  |NRQAAAAAABwAEHNh|
+00000140  62 48 52 41 62 6d 39 30  59 58 52 70 62 32 35 7a  |bHRAbm90YXRpb25z|
+00000150  0a 4c 6d 39 77 5a 57 35  77 5a 33 42 71 63 79 35  |.Lm9wZW5wZ3Bqcy5|
+00000160  76 63 6d 64 64 78 66 79  6b 4c 4c 54 71 6d 54 44  |vcmddxfykLLTqmTD|
+00000170  37 47 2f 6a 48 47 7a 59  2b 46 69 45 45 58 70 59  |7G/jHGzY+FiEEXpY|
+00000180  71 4e 31 6b 4e 68 56 74  47 66 53 41 70 53 55 65  |qN1kNhVtGfSApSUe|
+00000190  6d 0a 41 78 6c 68 48 6d  38 41 41 48 30 68 41 51  |m.AxlhHm8AAH0hAQ|
+000001a0  44 49 36 6f 47 78 76 6b  31 73 47 71 49 63 75 33  |DI6oGxvk1sGqIcu3|
+000001b0  41 62 76 73 37 44 4d 2f  37 45 61 6c 78 6b 4e 44  |Abvs7DM/7EalxkND|
+000001c0  36 31 34 4e 37 35 46 43  67 2b 79 77 45 41 70 50  |614N75FCg+ywEApP|
+000001d0  36 4c 0a 6f 49 76 72 5a  55 67 45 77 38 44 55 62  |6L.oIvrZUgEw8DUb|
+000001e0  39 4a 65 4b 37 54 43 68  4f 69 36 50 55 41 48 55  |9JeK7TChOi6PUAHU|
+000001f0  46 59 37 35 46 70 36 68  41 45 3d 0a 3d 6e 66 58  |FY75Fp6hAE=.=nfX|
+00000200  35 0a 2d 2d 2d 2d 2d 45  4e 44 20 50 47 50 20 53  |5.-----END PGP S|
+00000210  49 47 4e 41 54 55 52 45  2d 2d 2d 2d 2d d2 0c 28  |IGNATURE-----..(|
+00000220  35 45 39 36 32 41 33 37  35 39 30 44 38 35 35 42  |5E962A37590D855B|
+00000230  34 36 37 44 32 30 32 39  34 39 34 37 41 36 30 33  |467D20294947A603|
+00000240  31 39 36 31 31 45 36 46  da 0c d4 03 2d 2d 2d 2d  |19611E6F....----|
+00000250  2d 42 45 47 49 4e 20 50  47 50 20 50 55 42 4c 49  |-BEGIN PGP PUBLI|
+00000260  43 20 4b 45 59 20 42 4c  4f 43 4b 2d 2d 2d 2d 2d  |C KEY BLOCK-----|
+00000270  0a 0a 78 6a 4d 45 61 73  31 30 67 42 59 4a 4b 77  |..xjMEas10gBYJKw|
+00000280  59 42 42 41 48 61 52 77  38 42 41 51 64 41 70 71  |YBBAHaRw8BAQdApq|
+00000290  33 32 53 79 4f 72 4e 75  76 31 4d 71 53 55 66 45  |32SyOrNuv1MqSUfE|
+000002a0  44 67 74 42 7a 6e 4b 59  32 39 38 2f 30 47 46 55  |DgtBznKY298/0GFU|
+000002b0  76 6b 0a 6c 6f 42 71 35  69 6e 4e 4b 47 31 6d 5a  |vk.loBq5inNKG1mZ|
+000002c0  58 49 67 64 47 56 7a 64  43 42 32 5a 57 4e 30 62  |XIgdGVzdCB2ZWN0b|
+000002d0  33 49 67 50 48 52 6c 63  33 51 74 64 6d 56 6a 64  |3IgPHRlc3QtdmVjd|
+000002e0  47 39 79 51 47 31 6d 5a  58 49 75 64 47 56 7a 64  |G9yQG1mZXIudGVzd|
+000002f0  44 37 43 0a 76 51 51 54  46 67 67 41 62 77 57 43  |D7C.vQQTFggAbwWC|
+00000300  61 73 31 30 67 41 49 4c  42 77 6b 51 53 55 65 6d  |as10gAILBwkQSUem|
+00000310  41 78 6c 68 48 6d 38 31  46 41 41 41 41 41 41 41  |AxlhHm81FAAAAAAA|
+00000320  48 41 41 51 63 32 46 73  64 45 42 75 62 33 52 68  |HAAQc2FsdEBub3Rh|
+00000330  64 47 6c 76 0a 62 6e 4d  75 62 33 42 6c 62 6e 42  |dGlv.bnMub3BlbnB|
+00000340  6e 63 47 70 7a 4c 6d 39  79 5a 30 79 62 6c 73 34  |ncGpzLm9yZ0ybls4|
+00000350  71 53 43 6d 77 4a 64 45  49 45 6d 4c 72 54 55 45  |qSCmwJdEIEmLrTUE|
+00000360  43 46 51 67 43 46 67 41  43 47 51 45 43 6d 77 4d  |CFQgCFgACGQECmwM|
+00000370  43 48 67 45 57 0a 49 51  52 65 6c 69 6f 33 57 51  |CHgEW.IQRelio3WQ|
+00000380  32 46 57 30 5a 39 49 43  6c 4a 52 36 59 44 47 57  |2FW0Z9IClJR6YDGW|
+00000390  45 65 62 77 41 41 31 67  73 41 2f 32 61 2f 50 58  |EebwAA1gsA/2a/PX|
+000003a0  68 6c 44 49 4b 78 67 32  79 4e 30 4c 58 39 48 68  |hlDIKxg2yN0LX9Hh|
+000003b0  49 44 43 54 7a 69 0a 49  32 43 53 71 44 74 2b 34  |IDCTzi.I2CSqDt+4|
+000003c0  62 2f 73 7a 6d 59 64 41  51 43 66 2b 63 6a 6b 4f  |b/szmYdAQCf+cjkO|
+000003d0  71 6c 34 4c 47 5a 41 43  6c 2f 73 2f 2f 6d 31 71  |ql4LGZACl/s//m1q|
+000003e0  62 74 6e 2f 4a 31 46 45  53 52 56 73 4e 68 65 2f  |btn/J1FESRVsNhe/|
+000003f0  76 6d 69 41 67 3d 3d 0a  3d 56 6b 4f 33 0a 2d 2d  |vmiAg==.=VkO3.--|
+00000400  2d 2d 2d 45 4e 44 20 50  47 50 20 50 55 42 4c 49  |---END PGP PUBLI|
+00000410  43 20 4b 45 59 20 42 4c  4f 43 4b 2d 2d 2d 2d 2d  |C KEY BLOCK-----|
+```
 
 ## MIME Type
 

internal/cli/freshen.go

@@ -28,9 +28,15 @@ const (
 	scanProgressInterval = 100
 )
 
-// errEntryMissingMtime indicates a manifest entry that carries no
+var (
-// modification time where one is required to carry it forward unchanged.
+	// errEntryMissingMtime indicates a manifest entry that carries no
-var errEntryMissingMtime = errors.New("manifest entry has no mtime")
+	// modification time where one is required to carry it forward unchanged.
+	errEntryMissingMtime = errors.New("manifest entry has no mtime")
+
+	// errEntryMissingHash indicates a manifest entry that carries no hash
+	// to carry forward.
+	errEntryMissingHash = errors.New("manifest entry has no hash")
+)
 
 // FreshenStatus contains progress information for the freshen operation.
 type FreshenStatus struct {
@@ -644,6 +650,8 @@ func addFileToBuilder(
 // Entries reach this path only when recordEntry classified them as
 // unchanged, which requires a recorded mtime, so an absent mtime here is
 // an error rather than something to paper over with the Unix epoch.
+// Loading refuses an entry with no hash; one that got here anyway is an
+// error too, not a file to leave out of the manifest.
 func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
 	mtime, ok := entryMtime(entry)
 	if !ok {
@@ -651,7 +659,7 @@ func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
 	}
 
 	if len(entry.GetHashes()) == 0 {
-		return nil
+		return fmt.Errorf("%w: %s", errEntryMissingHash, entry.GetPath())
 	}
 
 	err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),

internal/cli/freshen_test.go

@@ -471,6 +471,24 @@ func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
 	assert.Contains(t, err.Error(), "file1.txt")
 }
 
+// TestFreshenAddExistingRejectsMissingHash pins that an entry with no hash
+// is an error rather than a file left out of the rebuilt manifest.
+func TestFreshenAddExistingRejectsMissingHash(t *testing.T) {
+	t.Parallel()
+
+	b := mfer.NewBuilder()
+	entry := &mfer.MFFilePath{
+		Path:  "data.bin",
+		Size:  8,
+		Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
+	}
+
+	err := addExistingToBuilder(b, entry)
+	require.ErrorIs(t, err, errEntryMissingHash)
+	assert.Contains(t, err.Error(), "data.bin")
+	assert.Zero(t, b.FileCount())
+}
+
 // TestFreshenAddExistingRejectsShortHash pins that an existing manifest
 // entry whose hash the builder refuses is reported as a problem with the
 // manifest, with the command that fixes it.

mfer/deserialize.go

@@ -27,6 +27,7 @@ var (
 	errUUIDMismatch         = errors.New("outer and inner UUID mismatch")
 	errInvalidFileFormat    = errors.New("invalid file format")
 	errInvalidManifestPath  = errors.New("invalid file entry")
+	errEntryMissingHash     = errors.New("file entry has no hash")
 	errDecodedTooLarge      = errors.New("too much memory needed")
 	errSignerNotSigningKey  = errors.New(
 		"signer is not the fingerprint of the key that made the signature")
@@ -293,7 +294,8 @@ func (m *manifest) deserializeInner() error {
 	// untrusted .mf. Reject loudly on the first offender rather than
 	// dropping entries, which would let a hostile manifest hide files from a
 	// check. A path listed twice is refused too: check would check the one
-	// file against both entries.
+	// file against both entries. So is an entry with no hash: there is
+	// nothing to check the file against.
 	seen := make(map[string]bool, len(m.pbInner.GetFiles()))
 
 	for _, f := range m.pbInner.GetFiles() {
@@ -302,6 +304,10 @@ func (m *manifest) deserializeInner() error {
 			return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
 		}
 
+		if len(f.GetHashes()) == 0 {
+			return fmt.Errorf("%w: %q", errEntryMissingHash, f.GetPath())
+		}
+
 		if seen[f.GetPath()] {
 			return fmt.Errorf("%w %q", errDuplicatePath, f.GetPath())
 		}

mfer/deserialize_path_test.go

@@ -165,6 +165,24 @@ func TestDeserializeRefusesPathListedTwice(t *testing.T) {
 	}
 }
 
+// A manifest holding a file entry with no hash is refused as it is loaded,
+// naming the path: there is nothing to check the file against.
+func TestDeserializeRefusesEntryWithNoHash(t *testing.T) {
+	t.Parallel()
+
+	id := uuid.NewV4()
+	inner, err := proto.Marshal(&MFFile{
+		Version: MFFile_VERSION_ONE,
+		Files:   []*MFFilePath{{Path: "x", Size: 1}},
+		Uuid:    id[:],
+	})
+	require.NoError(t, err)
+
+	_, err = NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
+	require.ErrorIs(t, err, errEntryMissingHash)
+	require.EqualError(t, err, `file entry has no hash: "x"`)
+}
+
 // Entries of a path, an empty hash, an empty MIME type and empty modification
 // and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
 // and take 16 bytes plus the path to encode. A 37-character path makes that
@@ -229,6 +247,8 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
 
 	entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
 	entry = protowire.AppendString(entry, "a")
+	entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
+	entry = protowire.AppendBytes(entry, nil)
 	entry = append(entry, unknown...)
 
 	id := uuid.NewV4()

mfer/deserialize_test.go

@@ -52,3 +52,29 @@ func TestReadAtMost(t *testing.T) {
 		"file exceeds maximum allowed size of 65536 bytes")
 	assert.Equal(t, maxSize-1, input.Len(), "bytes left unread")
 }
+
+// TestFormatTestVector loads the test vector that docs/FORMAT.md shows,
+// verifies its signature, and checks the signed string and the signer the
+// document gives for it.
+func TestFormatTestVector(t *testing.T) {
+	t.Parallel()
+
+	const signer = "5E962A37590D855B467D20294947A60319611E6F"
+
+	m, err := NewManifestFromFile(&ManifestFromFileOptions{
+		Path: "testdata/test-vector.mf",
+	})
+	require.NoError(t, err)
+
+	signedString, err := m.signatureString()
+	require.NoError(t, err)
+	assert.Equal(t, "ZNAVSRFG-92197fd532bc40908aa9f86e5518b56c-"+
+		"12202353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352",
+		signedString)
+
+	keyFingerprint, err := verifySignature([]byte(signedString),
+		m.pbOuter.GetSignature(), m.pbOuter.GetSigningPubKey())
+	require.NoError(t, err)
+	assert.Equal(t, signer, keyFingerprint)
+	assert.Equal(t, signer, string(m.pbOuter.GetSigner()))
+}

mfer/openpgp.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/ProtonMail/go-crypto/openpgp"
 	"github.com/ProtonMail/go-crypto/openpgp/armor"
@@ -242,10 +243,10 @@ func fingerprint(key *openpgp.Entity) string {
 	return strings.ToUpper(hex.EncodeToString(key.PrimaryKey.Fingerprint))
 }
 
-// verifySignature checks that signature is one good OpenPGP signature
+// verifySignature checks that signature is one good, unexpired OpenPGP
-// over data, made by the one primary key in pubKey or one of its subkeys,
+// signature over data, made by the one primary key in pubKey or one of its
-// and returns that primary key's fingerprint. signature and pubKey may each
+// subkeys, and returns that primary key's fingerprint. signature and pubKey
-// be armored or binary.
+// may each be armored or binary.
 func verifySignature(data, signature, pubKey []byte) (string, error) {
 	key, err := readOneKey(pubKey, "embedded public key block", true)
 	if err != nil {
@@ -266,10 +267,16 @@ func verifySignature(data, signature, pubKey []byte) (string, error) {
 		return "", fmt.Errorf("%w, found %d", errNotOneSignature, sigs)
 	}
 
-	_, err = openpgp.CheckDetachedSignature(openpgp.EntityList{key},
+	sig, _, err := openpgp.VerifyDetachedSignature(openpgp.EntityList{key},
 		bytes.NewReader(data), bytes.NewReader(sigData), nil)
 	// A manifest outlives its signing key, so a signature by a key that
-	// has expired since is still good.
+	// has expired since is still good, unless the signature has expired
+	// itself. The library does not check that once it finds the key
+	// expired.
+	if errors.Is(err, pgperrors.ErrKeyExpired) && sig.SigExpired(time.Now()) {
+		err = pgperrors.ErrSignatureExpired
+	}
+
 	if err != nil && !errors.Is(err, pgperrors.ErrKeyExpired) {
 		return "", fmt.Errorf("verify signature: %w", err)
 	}

mfer/openpgp_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ProtonMail/go-crypto/openpgp"
 	"github.com/ProtonMail/go-crypto/openpgp/armor"
 	"github.com/ProtonMail/go-crypto/openpgp/elgamal"
+	pgperrors "github.com/ProtonMail/go-crypto/openpgp/errors"
 	"github.com/ProtonMail/go-crypto/openpgp/packet"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
@@ -416,8 +417,10 @@ func TestVerifySignature(t *testing.T) {
 	assert.Error(t, err)
 }
 
-// TestVerifySignatureKeyExpiredSince verifies a signature made in 2020 by
+// TestVerifySignatureKeyExpiredSince verifies two signatures made in 2020
-// a key that expired a day after it was made. The signature is still good.
+// by a key that expired a day after it was made. A signature with no
+// expiry time is still good. One that expired an hour after it was made is
+// refused, although the key has expired too.
 func TestVerifySignatureKeyExpiredSince(t *testing.T) {
 	t.Parallel()
 
@@ -429,9 +432,14 @@ func TestVerifySignatureKeyExpiredSince(t *testing.T) {
 	key := newTestKey(t, config)
 	data := []byte("signed in 2020")
 
-	var sig bytes.Buffer
+	var sig, expiringSig bytes.Buffer
 
 	require.NoError(t, openpgp.ArmoredDetachSign(&sig, key, bytes.NewReader(data), config))
+	require.NoError(t, openpgp.ArmoredDetachSign(&expiringSig, key,
+		bytes.NewReader(data), &packet.Config{
+			Time:            config.Time,
+			SigLifetimeSecs: uint32(time.Hour.Seconds()),
+		}))
 
 	pubKey, err := armoredPublicKey(key)
 	require.NoError(t, err)
@@ -439,6 +447,9 @@ func TestVerifySignatureKeyExpiredSince(t *testing.T) {
 	signer, err := verifySignature(data, sig.Bytes(), pubKey)
 	require.NoError(t, err)
 	assert.Equal(t, fingerprint(key), signer)
+
+	_, err = verifySignature(data, expiringSig.Bytes(), pubKey)
+	require.ErrorIs(t, err, pgperrors.ErrSignatureExpired)
 }
 
 func TestManifestSignatureVerification(t *testing.T) {