2026-10-19 20:52:01 +02:00
13 changed files with 159 additions and 113 deletions
@@ -196,8 +196,7 @@ The manifest file would do several important things:
  a signature and a public key alongside in a single file
 - contain a list of files, each with a relative path to the manifest
 - contain manifest timestamp
- contain ctime/mtime information for files so that file metadata can be
-  preserved
+- contain mtime information for files so that file metadata can be preserved
 - contain cryptographic checksums in several different algorithms for each file
    - probably encoded with multihash to indicate algo + hash
    - sha256 at the minimum
@@ -369,8 +368,8 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
    - (fetches `https://example.com/manifestdirectory/index.mf`, downloads and
      checksums all files, resumes any that exist locally already)
 - `mfer fetch https://example.com/manifestdirectory/`
- a command line option to zero/omit mtime/ctime, as well as manifest timestamp,
-  and sort all directory listings so that manifest file generation is
+- a command line option to zero/omit mtime, as well as manifest timestamp, and
+  sort all directory listings so that manifest file generation is
  deterministic/reproducible
 - URL format
  `mfer fetch https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2`
@@ -9,8 +9,11 @@ including their paths, sizes, and cryptographic checksums. It supports optional
 OpenPGP signatures for integrity verification and optional timestamps and file
 permissions for metadata preservation.

-Nothing goes in the 1.0 manifest that 1.0 does not read or write: no field is
-reserved or kept for later use.
+Nothing goes in the 1.0 manifest that 1.0 does not read or write, and no field
+is kept for later use. [`mfer/mf.proto`](../mfer/mf.proto) reserves the field
+numbers 301 and 303 and the names `mimeType` and `ctime` in `MFFilePath`, so
+that a later version cannot reuse them by accident. No field uses them, and
+readers ignore them like any field they do not know.

 ## File Structure

@@ -58,11 +61,11 @@ enforce a decompression size limit to prevent decompression bombs. The reference
 implementation limits decompressed size to 256 MiB. It writes zstd frames with a
 window of at most 8 MiB, the largest window the zstd format recommends decoders
 support, and refuses frames that ask for a larger one. It also refuses an inner
-message whose file entries, hashes, timestamps and MIME types, counted at 176,
-112, 64 and 16 bytes each, add up to more than 8 times its size. It refuses a
-manifest file larger than 258 MiB without reading the rest of it: zstd's worst
-case grows a 256 MiB inner message by 1/256 to 257 MiB, and the last MiB is room
-for the signature, the signing key and the other outer fields.
+message whose file entries, hashes and timestamps, counted at 160, 112 and 64
+bytes each, add up to more than 8 times its size. It refuses a manifest file
+larger than 258 MiB without reading the rest of it: zstd's worst case grows a
+256 MiB inner message by 1/256 to 257 MiB, and the last MiB is room for the
+signature, the signing key and the other outer fields.

 ## Inner Message (`MFFile`)

@@ -80,15 +83,13 @@ After decompressing `innerMessage`, the result is a serialized `MFFile`

 Each file entry contains:

-| Field      | Number | Type                      | Description                                                  |
-| ---------- | ------ | ------------------------- | ------------------------------------------------------------ |
-| `path`     | 1      | string                    | Relative file path (see Path Rules)                          |
-| `size`     | 2      | int64                     | File size in bytes                                           |
-| `hashes`   | 3      | repeated `MFFileChecksum` | Must include exactly one SHA-256 multihash (see Hash Format) |
-| `mimeType` | 301    | string (optional)         | MIME type                                                    |
-| `mtime`    | 302    | Timestamp (optional)      | Modification time                                            |
-| `ctime`    | 303    | Timestamp (optional)      | Change time (inode metadata change)                          |
-| `mode`     | 304    | uint32                    | Permission bits (see File Mode)                              |
+| Field    | Number | Type                      | Description                                                  |
+| -------- | ------ | ------------------------- | ------------------------------------------------------------ |
+| `path`   | 1      | string                    | Relative file path (see Path Rules)                          |
+| `size`   | 2      | int64                     | File size in bytes                                           |
+| `hashes` | 3      | repeated `MFFileChecksum` | Must include exactly one SHA-256 multihash (see Hash Format) |
+| `mtime`  | 302    | Timestamp (optional)      | Modification time                                            |
+| `mode`   | 304    | uint32                    | Permission bits (see File Mode)                              |

 ## File Mode

@@ -17,7 +17,6 @@ type ExportEntry struct {
 	Size   int64    `json:"size"`
 	Hashes []string `json:"hashes"`
 	Mtime  *string  `json:"mtime,omitempty"`
-	Ctime  *string  `json:"ctime,omitempty"`
 	Mode   string   `json:"mode"` // octal, "0000" when none was recorded
 }

@@ -61,12 +60,6 @@ func (mfa *CLIApp) exportManifestOperation(
 			entry.Mtime = &t
 		}

-		if f.GetCtime() != nil {
-			t := time.Unix(f.GetCtime().GetSeconds(), int64(f.GetCtime().GetNanos())).
-				UTC().Format(time.RFC3339Nano)
-			entry.Ctime = &t
-		}
-
 		entries = append(entries, entry)
 	}

@@ -4,8 +4,10 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"maps"
 	"net/http"
 	"net/http/httptest"
+	"slices"
 	"testing"

 	"github.com/spf13/afero"
@@ -84,6 +86,36 @@ func TestExportManifestOperation(t *testing.T) {
 	assert.True(t, pathSet["sub/file.txt"])
 }

+// export prints the fields of a file entry and nothing else, so no ctime.
+func TestExportPrintsOnlyFileEntryFields(t *testing.T) {
+	t.Parallel()
+
+	manifestData := buildTestManifest(t, map[string][]byte{"entry.txt": []byte("aaa")})
+
+	fs := afero.NewMemMapFs()
+	require.NoError(t, afero.WriteFile(fs, "/test.mf", manifestData, 0o644))
+
+	var stdout, stderr bytes.Buffer
+
+	exitCode := runCLI(&RunOptions{
+		Appname: testApp,
+		Args:    []string{testApp, testCmdExport, "/test.mf"},
+		Stdin:   &bytes.Buffer{},
+		Stdout:  &stdout,
+		Stderr:  &stderr,
+		Fs:      fs,
+	})
+
+	require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
+
+	var entries []map[string]json.RawMessage
+	require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries))
+	require.Len(t, entries, 1)
+	assert.Equal(t,
+		[]string{"hashes", "mode", "mtime", "path", "size"},
+		slices.Sorted(maps.Keys(entries[0])))
+}
+
 func TestExportFromHTTPURL(t *testing.T) {
 	t.Parallel()

@@ -21,25 +21,22 @@ const (

 	// Numbers in mf.proto of MFFile.files and of the MFFilePath fields
 	// that decoding sets aside a fixed amount of memory for.
-	filesFieldNumber    = 101
-	hashesFieldNumber   = 3
-	mimeTypeFieldNumber = 301
-	mtimeFieldNumber    = 302
-	ctimeFieldNumber    = 303
+	filesFieldNumber  = 101
+	hashesFieldNumber = 3
+	mtimeFieldNumber  = 302

-	// Bytes decoding sets aside for each file entry, hash, timestamp and
-	// MIME type, however short its encoding. checkDecodedSize refuses an
-	// inner message for which these add up to more than maxDecodedGrowth
-	// times its size. The mode is held in the file entry itself.
-	decodedFileEntrySize = 176
+	// Bytes decoding sets aside for each file entry, hash and timestamp,
+	// however short its encoding. checkDecodedSize refuses an inner
+	// message for which these add up to more than maxDecodedGrowth times
+	// its size. The mode is held in the file entry itself.
+	decodedFileEntrySize = 160
 	decodedHashSize      = 112
 	decodedTimestampSize = 64
-	decodedMIMETypeSize  = 16

 	// Each file entry mfer writes holds a path of at least one byte, a
 	// multihash at least as long as SHA-256's 34 bytes (AddFileWithHash
 	// refuses shorter ones) and a modification time: at least 47 bytes,
-	// counted at 352. So its manifests add up to at most about 7.49 times
-	// their size, and this limit is about 7% above that.
+	// counted at 336. So its manifests add up to at most about 7.15 times
+	// their size, and this limit is about 12% above that.
 	maxDecodedGrowth = 8
 )
@@ -161,10 +161,10 @@ func (m *manifest) decompressInner() ([]byte, error) {
 }

 // checkDecodedSize refuses an encoded inner message whose file entries,
-// hashes, timestamps and MIME types would take more than maxDecodedGrowth
-// times its size to decode. Decoding sets aside a fixed amount for each,
-// however short its encoding, so a message of empty ones would take about
-// 50 times its size.
+// hashes and timestamps would take more than maxDecodedGrowth times its
+// size to decode. Decoding sets aside a fixed amount for each, however
+// short its encoding, so a message of empty ones would take about 50 times
+// its size.
 func checkDecodedSize(inner []byte) error {
 	limit := maxDecodedGrowth * int64(len(inner))

@@ -194,14 +194,10 @@ func checkDecodedSize(inner []byte) error {
 				return add(decodedHashSize)
 			}

-			if num == mtimeFieldNumber || num == ctimeFieldNumber {
+			if num == mtimeFieldNumber {
 				return add(decodedTimestampSize)
 			}

-			if num == mimeTypeFieldNumber {
-				return add(decodedMIMETypeSize)
-			}
-
 			return nil
 		})
 	})
@@ -48,8 +48,8 @@ func FuzzNewManifestFromReader(f *testing.F) {
 		// It also keeps a few copies of its input. Buffers grow by
 		// copying, so reaching those sizes allocates up to about six times
 		// them in total. Decoding the decompressed data takes up to
-		// maxDecodedGrowth times its size for file entries, hashes,
-		// timestamps and MIME types, and drops fields it does not know.
+		// maxDecodedGrowth times its size for file entries, hashes and
+		// timestamps, and drops fields it does not know.
 		// The strings and bytes it copies out of it, such as many one-byte
 		// values in one hash, take up to about five times more under the
 		// race detector, which pads every small copy to 16 bytes, and about
@@ -295,14 +295,17 @@ func TestDeserializeRefusesModeOutsidePermissionBits(t *testing.T) {
 	}
 }

-// Entries of a path, a SHA-256 hash, an empty MIME type and empty
-// modification and change times are counted at 432 bytes each (176 + 112 +
-// 16 + 64 + 64) and take 52 bytes plus the path to encode. A one-character
-// path makes that 53 bytes, about 8.2 times: refused, and leaving any one of
-// the five uncounted, even the MIME type, brings it under 8. A two-character
-// path makes it 54 bytes, 8 times exactly: loaded. There is one entry for
-// each base-36 digit, and each entry's path is its number in base 36, padded
-// with zeros to that length, since a manifest lists a path only once.
+// Entries of a path, a SHA-256 hash, an empty second hash and an empty
+// modification time are counted at 448 bytes each (160 + 112 + 112 + 64)
+// and take 48 bytes plus the path to encode. A 7-character path makes that
+// 55 bytes, about 8.1 times: refused, and leaving any one of the four
+// uncounted, even the modification time, brings it under 8. An 8-character
+// path makes it 56 bytes, 8 times exactly: loaded. Without the second hash
+// an entry is never refused: it takes at least 47 bytes and is counted at
+// 336. There is one entry for each base-36 digit, enough that the 22 bytes
+// of the version and UUID change neither outcome, and each entry's path is
+// its number in base 36, padded with zeros to that length, since a manifest
+// lists a path only once.
 func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
 	t.Parallel()

@@ -316,8 +319,8 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
 		pathLen int
 		refused bool
 	}{
-		{1, true},
-		{2, false},
+		{7, true},
+		{8, false},
 	}

 	for _, tt := range tests {
@@ -338,12 +341,10 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
 				entry = protowire.AppendString(entry, path)
 				entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
 				entry = protowire.AppendBytes(entry, checksum)
-				entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
+				entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
 				entry = protowire.AppendBytes(entry, nil)
 				entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
 				entry = protowire.AppendBytes(entry, nil)
-				entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
-				entry = protowire.AppendBytes(entry, nil)

 				inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
 				inner = protowire.AppendBytes(inner, entry)
@@ -399,6 +400,44 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
 	assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
 }

+// A file entry holding fields 301 and 303, which mf.proto reserves, loads,
+// and the loaded entry keeps neither.
+func TestDeserializeDropsReservedFields(t *testing.T) {
+	t.Parallel()
+
+	hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
+	require.NoError(t, err)
+
+	checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
+	require.NoError(t, err)
+
+	timestamp, err := proto.Marshal(&Timestamp{Seconds: 1})
+	require.NoError(t, err)
+
+	entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
+	entry = protowire.AppendString(entry, "a")
+	entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
+	entry = protowire.AppendBytes(entry, checksum)
+	entry = protowire.AppendTag(entry, 301, protowire.BytesType) // reserved
+	entry = protowire.AppendString(entry, "text/plain")
+	entry = protowire.AppendTag(entry, 303, protowire.BytesType) // reserved
+	entry = protowire.AppendBytes(entry, timestamp)
+
+	id := uuid.NewV4()
+	inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
+	inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
+	inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
+	inner = protowire.AppendBytes(inner, entry)
+	inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
+	inner = protowire.AppendBytes(inner, id[:])
+
+	m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
+	require.NoError(t, err)
+	require.Len(t, m.Files(), 1)
+	assert.Equal(t, "a", m.Files()[0].GetPath())
+	assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
+}
+
 // Many empty files with names of at most three characters and modification
 // times at the epoch make about the densest manifest mfer writes: it takes
 // about 7 times its size to decode, and still loads. A signature would not
@@ -340,9 +340,7 @@ type MFFilePath struct {
 	// gotta have at least one:
 	Hashes []*MFFileChecksum `protobuf:"bytes,3,rep,name=hashes,proto3" json:"hashes,omitempty"`
 	// optional per-file metadata
-	MimeType *string    `protobuf:"bytes,301,opt,name=mimeType,proto3,oneof" json:"mimeType,omitempty"`
-	Mtime    *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"`
-	Ctime    *Timestamp `protobuf:"bytes,303,opt,name=ctime,proto3,oneof" json:"ctime,omitempty"`
+	Mtime *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"`
 	// permission bits, at most 0777; 0 when not recorded
 	Mode          uint32 `protobuf:"varint,304,opt,name=mode,proto3" json:"mode,omitempty"`
 	unknownFields protoimpl.UnknownFields
@@ -400,13 +398,6 @@ func (x *MFFilePath) GetHashes() []*MFFileChecksum {
 	return nil
 }

-func (x *MFFilePath) GetMimeType() string {
-	if x != nil && x.MimeType != nil {
-		return *x.MimeType
-	}
-	return ""
-}
-
 func (x *MFFilePath) GetMtime() *Timestamp {
 	if x != nil {
 		return x.Mtime
@@ -414,13 +405,6 @@ func (x *MFFilePath) GetMtime() *Timestamp {
 	return nil
 }

-func (x *MFFilePath) GetCtime() *Timestamp {
-	if x != nil {
-		return x.Ctime
-	}
-	return nil
-}
-
 func (x *MFFilePath) GetMode() uint32 {
 	if x != nil {
 		return x.Mode
@@ -573,21 +557,16 @@ const file_mf_proto_rawDesc = "" +
 	"\n" +
 	"_signatureB\t\n" +
 	"\a_signerB\x10\n" +
-	"\x0e_signingPubKey\"\x85\x02\n" +
+	"\x0e_signingPubKey\"\xc5\x01\n" +
 	"\n" +
 	"MFFilePath\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" +
 	"\x04size\x18\x02 \x01(\x03R\x04size\x12'\n" +
-	"\x06hashes\x18\x03 \x03(\v2\x0f.MFFileChecksumR\x06hashes\x12 \n" +
-	"\bmimeType\x18\xad\x02 \x01(\tH\x00R\bmimeType\x88\x01\x01\x12&\n" +
+	"\x06hashes\x18\x03 \x03(\v2\x0f.MFFileChecksumR\x06hashes\x12&\n" +
 	"\x05mtime\x18\xae\x02 \x01(\v2\n" +
-	".TimestampH\x01R\x05mtime\x88\x01\x01\x12&\n" +
-	"\x05ctime\x18\xaf\x02 \x01(\v2\n" +
-	".TimestampH\x02R\x05ctime\x88\x01\x01\x12\x13\n" +
-	"\x04mode\x18\xb0\x02 \x01(\rR\x04modeB\v\n" +
-	"\t_mimeTypeB\b\n" +
-	"\x06_mtimeB\b\n" +
-	"\x06_ctime\".\n" +
+	".TimestampH\x00R\x05mtime\x88\x01\x01\x12\x13\n" +
+	"\x04mode\x18\xb0\x02 \x01(\rR\x04modeB\b\n" +
+	"\x06_mtimeJ\x06\b\xad\x02\x10\xae\x02J\x06\b\xaf\x02\x10\xb0\x02R\bmimeTypeR\x05ctime\".\n" +
 	"\x0eMFFileChecksum\x12\x1c\n" +
 	"\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xd6\x01\n" +
 	"\x06MFFile\x12)\n" +
@@ -631,15 +610,14 @@ var file_mf_proto_depIdxs = []int32{
 	1, // 1: MFFileOuter.compressionType:type_name -> MFFileOuter.CompressionType
 	6, // 2: MFFilePath.hashes:type_name -> MFFileChecksum
 	3, // 3: MFFilePath.mtime:type_name -> Timestamp
-	3, // 4: MFFilePath.ctime:type_name -> Timestamp
-	2, // 5: MFFile.version:type_name -> MFFile.Version
-	5, // 6: MFFile.files:type_name -> MFFilePath
-	3, // 7: MFFile.createdAt:type_name -> Timestamp
-	8, // [8:8] is the sub-list for method output_type
-	8, // [8:8] is the sub-list for method input_type
-	8, // [8:8] is the sub-list for extension type_name
-	8, // [8:8] is the sub-list for extension extendee
-	0, // [0:8] is the sub-list for field type_name
+	2, // 4: MFFile.version:type_name -> MFFile.Version
+	5, // 5: MFFile.files:type_name -> MFFilePath
+	3, // 6: MFFile.createdAt:type_name -> Timestamp
+	7, // [7:7] is the sub-list for method output_type
+	7, // [7:7] is the sub-list for method input_type
+	7, // [7:7] is the sub-list for extension type_name
+	7, // [7:7] is the sub-list for extension extendee
+	0, // [0:7] is the sub-list for field type_name
 }

 func init() { file_mf_proto_init() }
@@ -58,11 +58,13 @@ message MFFilePath {
    repeated MFFileChecksum hashes = 3;

    // optional per-file metadata
-    optional string mimeType = 301;
    optional Timestamp mtime = 302;
-    optional Timestamp ctime = 303;
    // permission bits, at most 0777; 0 when not recorded
    uint32 mode = 304;
+
+    // used by no field, so that a later version cannot reuse them by accident
+    reserved 301, 303;
+    reserved "mimeType", "ctime";
 }

 message MFFileChecksum {
@@ -1 +1 @@
-1668b2f6b8639a9213873474e58dbacb3a4ef77cc27c42253cad8dd076e5f206  mf.proto
+16a2e6b39986d02aa1c7c01dc6b97c9d4c30db7a9d000056bd5985a3fa7c8089  mf.proto
@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"

+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/reflect/protoreflect"
 	"sneak.berlin/go/mfer/mfer"
@@ -35,13 +36,11 @@ func TestFileEntryFieldsMatchSpec(t *testing.T) {
 	t.Parallel()

 	want := map[string]protoreflect.FieldNumber{
-		"path":     1,
-		"size":     2,
-		"hashes":   3,
-		"mimeType": 301,
-		"mtime":    302,
-		"ctime":    303,
-		"mode":     304,
+		"path":   1,
+		"size":   2,
+		"hashes": 3,
+		"mtime":  302,
+		"mode":   304,
 	}

 	got := map[string]protoreflect.FieldNumber{}
@@ -53,3 +52,16 @@ func TestFileEntryFieldsMatchSpec(t *testing.T) {

 	require.Equal(t, want, got)
 }
+
+// A file entry reserves the numbers 301 and 303 and the names mimeType and
+// ctime, as docs/FORMAT.md says, so that no field can take them.
+func TestFileEntryReservesMIMETypeAndCtime(t *testing.T) {
+	t.Parallel()
+
+	desc := (&mfer.MFFilePath{}).ProtoReflect().Descriptor()
+
+	assert.True(t, desc.ReservedRanges().Has(301))
+	assert.True(t, desc.ReservedRanges().Has(303))
+	assert.True(t, desc.ReservedNames().Has("mimeType"))
+	assert.True(t, desc.ReservedNames().Has("ctime"))
+}
@@ -76,7 +76,6 @@ type FileEntry struct {
 	AbsPath AbsFilePath // Absolute path (used for reading file content)
 	Size    FileSize    // File size in bytes
 	Mtime   ModTime     // Last modification time
-	Ctime   time.Time   // Creation time (platform-dependent)
 	Mode    fs.FileMode // Permission bits (Perm() of the file's mode)
 }

@@ -498,8 +497,6 @@ func (s *Scanner) enumerateFileWithInfo(
 		Size:    FileSize(info.Size()),
 		Mtime:   ModTime(info.ModTime()),
 		Mode:    info.Mode().Perm(),
-		// Note: Ctime not available from fs.FileInfo on all platforms
-		// Will need platform-specific code to extract it
 	}

 	s.mu.Lock()