diff --git a/README.md b/README.md index 172c0a9..b696e4a 100644 --- a/README.md +++ b/README.md @@ -196,8 +196,7 @@ The manifest file would do several important things: a signature and a public key alongside in a single file - contain a list of files, each with a relative path to the manifest - contain manifest timestamp -- contain ctime/mtime information for files so that file metadata can be - preserved +- contain mtime information for files so that file metadata can be preserved - contain cryptographic checksums in several different algorithms for each file - probably encoded with multihash to indicate algo + hash - sha256 at the minimum @@ -369,8 +368,8 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues). - (fetches `https://example.com/manifestdirectory/index.mf`, downloads and checksums all files, resumes any that exist locally already) - `mfer fetch https://example.com/manifestdirectory/` -- a command line option to zero/omit mtime/ctime, as well as manifest timestamp, - and sort all directory listings so that manifest file generation is +- a command line option to zero/omit mtime, as well as manifest timestamp, and + sort all directory listings so that manifest file generation is deterministic/reproducible - URL format `mfer fetch https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2` diff --git a/docs/FORMAT.md b/docs/FORMAT.md index 5ed503e..e3ee620 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -9,8 +9,11 @@ including their paths, sizes, and cryptographic checksums. It supports optional OpenPGP signatures for integrity verification and optional timestamps and file permissions for metadata preservation. -Nothing goes in the 1.0 manifest that 1.0 does not read or write: no field is -reserved or kept for later use. +Nothing goes in the 1.0 manifest that 1.0 does not read or write, and no field +is kept for later use. [`mfer/mf.proto`](../mfer/mf.proto) reserves the field +numbers 301 and 303 and the names `mimeType` and `ctime` in `MFFilePath`, so +that a later version cannot reuse them by accident. No field uses them, and +readers ignore them like any field they do not know. ## File Structure @@ -58,11 +61,11 @@ enforce a decompression size limit to prevent decompression bombs. The reference implementation limits decompressed size to 256 MiB. It writes zstd frames with a window of at most 8 MiB, the largest window the zstd format recommends decoders support, and refuses frames that ask for a larger one. It also refuses an inner -message whose file entries, hashes, timestamps and MIME types, counted at 176, -112, 64 and 16 bytes each, add up to more than 8 times its size. It refuses a -manifest file larger than 258 MiB without reading the rest of it: zstd's worst -case grows a 256 MiB inner message by 1/256 to 257 MiB, and the last MiB is room -for the signature, the signing key and the other outer fields. +message whose file entries, hashes and timestamps, counted at 176, 112 and 64 +bytes each, add up to more than 8 times its size. It refuses a manifest file +larger than 258 MiB without reading the rest of it: zstd's worst case grows a +256 MiB inner message by 1/256 to 257 MiB, and the last MiB is room for the +signature, the signing key and the other outer fields. ## Inner Message (`MFFile`) @@ -80,15 +83,13 @@ After decompressing `innerMessage`, the result is a serialized `MFFile` Each file entry contains: -| Field | Number | Type | Description | -| ---------- | ------ | ------------------------- | ------------------------------------------------------------ | -| `path` | 1 | string | Relative file path (see Path Rules) | -| `size` | 2 | int64 | File size in bytes | -| `hashes` | 3 | repeated `MFFileChecksum` | Must include exactly one SHA-256 multihash (see Hash Format) | -| `mimeType` | 301 | string (optional) | MIME type | -| `mtime` | 302 | Timestamp (optional) | Modification time | -| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) | -| `mode` | 304 | uint32 | Permission bits (see File Mode) | +| Field | Number | Type | Description | +| -------- | ------ | ------------------------- | ------------------------------------------------------------ | +| `path` | 1 | string | Relative file path (see Path Rules) | +| `size` | 2 | int64 | File size in bytes | +| `hashes` | 3 | repeated `MFFileChecksum` | Must include exactly one SHA-256 multihash (see Hash Format) | +| `mtime` | 302 | Timestamp (optional) | Modification time | +| `mode` | 304 | uint32 | Permission bits (see File Mode) | ## File Mode diff --git a/internal/cli/export.go b/internal/cli/export.go index b343a9e..29e33fb 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -17,7 +17,6 @@ type ExportEntry struct { Size int64 `json:"size"` Hashes []string `json:"hashes"` Mtime *string `json:"mtime,omitempty"` - Ctime *string `json:"ctime,omitempty"` Mode string `json:"mode"` // octal, "0000" when none was recorded } @@ -61,12 +60,6 @@ func (mfa *CLIApp) exportManifestOperation( entry.Mtime = &t } - if f.GetCtime() != nil { - t := time.Unix(f.GetCtime().GetSeconds(), int64(f.GetCtime().GetNanos())). - UTC().Format(time.RFC3339Nano) - entry.Ctime = &t - } - entries = append(entries, entry) } diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go index ac76bc2..ac0ca61 100644 --- a/internal/cli/export_test.go +++ b/internal/cli/export_test.go @@ -4,8 +4,10 @@ import ( "bytes" "context" "encoding/json" + "maps" "net/http" "net/http/httptest" + "slices" "testing" "github.com/spf13/afero" @@ -84,6 +86,36 @@ func TestExportManifestOperation(t *testing.T) { assert.True(t, pathSet["sub/file.txt"]) } +// export prints the fields of a file entry and nothing else, so no ctime. +func TestExportPrintsOnlyFileEntryFields(t *testing.T) { + t.Parallel() + + manifestData := buildTestManifest(t, map[string][]byte{"entry.txt": []byte("aaa")}) + + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/test.mf", manifestData, 0o644)) + + var stdout, stderr bytes.Buffer + + exitCode := runCLI(&RunOptions{ + Appname: testApp, + Args: []string{testApp, testCmdExport, "/test.mf"}, + Stdin: &bytes.Buffer{}, + Stdout: &stdout, + Stderr: &stderr, + Fs: fs, + }) + + require.Equal(t, 0, exitCode, "stderr: %s", stderr.String()) + + var entries []map[string]json.RawMessage + require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries)) + require.Len(t, entries, 1) + assert.Equal(t, + []string{"hashes", "mode", "mtime", "path", "size"}, + slices.Sorted(maps.Keys(entries[0]))) +} + func TestExportFromHTTPURL(t *testing.T) { t.Parallel() diff --git a/mfer/constants.go b/mfer/constants.go index ef632d5..8cea06c 100644 --- a/mfer/constants.go +++ b/mfer/constants.go @@ -21,20 +21,17 @@ const ( // Numbers in mf.proto of MFFile.files and of the MFFilePath fields // that decoding sets aside a fixed amount of memory for. - filesFieldNumber = 101 - hashesFieldNumber = 3 - mimeTypeFieldNumber = 301 - mtimeFieldNumber = 302 - ctimeFieldNumber = 303 + filesFieldNumber = 101 + hashesFieldNumber = 3 + mtimeFieldNumber = 302 - // Bytes decoding sets aside for each file entry, hash, timestamp and - // MIME type, however short its encoding. checkDecodedSize refuses an - // inner message for which these add up to more than maxDecodedGrowth - // times its size. The mode is held in the file entry itself. + // Bytes decoding sets aside for each file entry, hash and timestamp, + // however short its encoding. checkDecodedSize refuses an inner + // message for which these add up to more than maxDecodedGrowth times + // its size. The mode is held in the file entry itself. decodedFileEntrySize = 176 decodedHashSize = 112 decodedTimestampSize = 64 - decodedMIMETypeSize = 16 // Each file entry mfer writes holds a path of at least one byte, a // multihash at least as long as SHA-256's 34 bytes (AddFileWithHash diff --git a/mfer/deserialize.go b/mfer/deserialize.go index 65a3511..d9bc2da 100644 --- a/mfer/deserialize.go +++ b/mfer/deserialize.go @@ -161,10 +161,10 @@ func (m *manifest) decompressInner() ([]byte, error) { } // checkDecodedSize refuses an encoded inner message whose file entries, -// hashes, timestamps and MIME types would take more than maxDecodedGrowth -// times its size to decode. Decoding sets aside a fixed amount for each, -// however short its encoding, so a message of empty ones would take about -// 50 times its size. +// hashes and timestamps would take more than maxDecodedGrowth times its +// size to decode. Decoding sets aside a fixed amount for each, however +// short its encoding, so a message of empty ones would take about 50 times +// its size. func checkDecodedSize(inner []byte) error { limit := maxDecodedGrowth * int64(len(inner)) @@ -194,14 +194,10 @@ func checkDecodedSize(inner []byte) error { return add(decodedHashSize) } - if num == mtimeFieldNumber || num == ctimeFieldNumber { + if num == mtimeFieldNumber { return add(decodedTimestampSize) } - if num == mimeTypeFieldNumber { - return add(decodedMIMETypeSize) - } - return nil }) }) diff --git a/mfer/deserialize_fuzz_test.go b/mfer/deserialize_fuzz_test.go index a6a97c7..81f3429 100644 --- a/mfer/deserialize_fuzz_test.go +++ b/mfer/deserialize_fuzz_test.go @@ -48,8 +48,8 @@ func FuzzNewManifestFromReader(f *testing.F) { // It also keeps a few copies of its input. Buffers grow by // copying, so reaching those sizes allocates up to about six times // them in total. Decoding the decompressed data takes up to - // maxDecodedGrowth times its size for file entries, hashes, - // timestamps and MIME types, and drops fields it does not know. + // maxDecodedGrowth times its size for file entries, hashes and + // timestamps, and drops fields it does not know. // The strings and bytes it copies out of it, such as many one-byte // values in one hash, take up to about five times more under the // race detector, which pads every small copy to 16 bytes, and about diff --git a/mfer/deserialize_path_test.go b/mfer/deserialize_path_test.go index 6a5259c..6f7b1e6 100644 --- a/mfer/deserialize_path_test.go +++ b/mfer/deserialize_path_test.go @@ -295,14 +295,17 @@ func TestDeserializeRefusesModeOutsidePermissionBits(t *testing.T) { } } -// Entries of a path, a SHA-256 hash, an empty MIME type and empty -// modification and change times are counted at 432 bytes each (176 + 112 + -// 16 + 64 + 64) and take 52 bytes plus the path to encode. A one-character -// path makes that 53 bytes, about 8.2 times: refused, and leaving any one of -// the five uncounted, even the MIME type, brings it under 8. A two-character -// path makes it 54 bytes, 8 times exactly: loaded. There is one entry for -// each base-36 digit, and each entry's path is its number in base 36, padded -// with zeros to that length, since a manifest lists a path only once. +// Entries of a path, a SHA-256 hash, an empty second hash and an empty +// modification time are counted at 464 bytes each (176 + 112 + 112 + 64) +// and take 48 bytes plus the path to encode. A 9-character path makes that +// 57 bytes, about 8.1 times: refused, and leaving any one of the four +// uncounted, even the modification time, brings it under 8. A 10-character +// path makes it 58 bytes, 8 times exactly: loaded. Without the second hash +// an entry is never refused: it takes at least 47 bytes and is counted at +// 352. There is one entry for each base-36 digit, enough that the 22 bytes +// of the version and UUID change neither outcome, and each entry's path is +// its number in base 36, padded with zeros to that length, since a manifest +// lists a path only once. func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) { t.Parallel() @@ -316,8 +319,8 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) { pathLen int refused bool }{ - {1, true}, - {2, false}, + {9, true}, + {10, false}, } for _, tt := range tests { @@ -338,12 +341,10 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) { entry = protowire.AppendString(entry, path) entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes entry = protowire.AppendBytes(entry, checksum) - entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType + entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes entry = protowire.AppendBytes(entry, nil) entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime entry = protowire.AppendBytes(entry, nil) - entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime - entry = protowire.AppendBytes(entry, nil) inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files inner = protowire.AppendBytes(inner, entry) @@ -399,6 +400,44 @@ func TestDeserializeDropsUnknownFields(t *testing.T) { assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown()) } +// A file entry holding fields 301 and 303, which mf.proto reserves, loads, +// and the loaded entry keeps neither. +func TestDeserializeDropsReservedFields(t *testing.T) { + t.Parallel() + + hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256) + require.NoError(t, err) + + checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash}) + require.NoError(t, err) + + timestamp, err := proto.Marshal(&Timestamp{Seconds: 1}) + require.NoError(t, err) + + entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path + entry = protowire.AppendString(entry, "a") + entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes + entry = protowire.AppendBytes(entry, checksum) + entry = protowire.AppendTag(entry, 301, protowire.BytesType) // reserved + entry = protowire.AppendString(entry, "text/plain") + entry = protowire.AppendTag(entry, 303, protowire.BytesType) // reserved + entry = protowire.AppendBytes(entry, timestamp) + + id := uuid.NewV4() + inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version + inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE)) + inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files + inner = protowire.AppendBytes(inner, entry) + inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid + inner = protowire.AppendBytes(inner, id[:]) + + m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner))) + require.NoError(t, err) + require.Len(t, m.Files(), 1) + assert.Equal(t, "a", m.Files()[0].GetPath()) + assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown()) +} + // Many empty files with names of at most three characters and modification // times at the epoch make about the densest manifest mfer writes: it takes // about 7 times its size to decode, and still loads. A signature would not diff --git a/mfer/mf.pb.go b/mfer/mf.pb.go index a80e409..c354c26 100644 --- a/mfer/mf.pb.go +++ b/mfer/mf.pb.go @@ -340,9 +340,7 @@ type MFFilePath struct { // gotta have at least one: Hashes []*MFFileChecksum `protobuf:"bytes,3,rep,name=hashes,proto3" json:"hashes,omitempty"` // optional per-file metadata - MimeType *string `protobuf:"bytes,301,opt,name=mimeType,proto3,oneof" json:"mimeType,omitempty"` - Mtime *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"` - Ctime *Timestamp `protobuf:"bytes,303,opt,name=ctime,proto3,oneof" json:"ctime,omitempty"` + Mtime *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"` // permission bits, at most 0777; 0 when not recorded Mode uint32 `protobuf:"varint,304,opt,name=mode,proto3" json:"mode,omitempty"` unknownFields protoimpl.UnknownFields @@ -400,13 +398,6 @@ func (x *MFFilePath) GetHashes() []*MFFileChecksum { return nil } -func (x *MFFilePath) GetMimeType() string { - if x != nil && x.MimeType != nil { - return *x.MimeType - } - return "" -} - func (x *MFFilePath) GetMtime() *Timestamp { if x != nil { return x.Mtime @@ -414,13 +405,6 @@ func (x *MFFilePath) GetMtime() *Timestamp { return nil } -func (x *MFFilePath) GetCtime() *Timestamp { - if x != nil { - return x.Ctime - } - return nil -} - func (x *MFFilePath) GetMode() uint32 { if x != nil { return x.Mode @@ -573,21 +557,16 @@ const file_mf_proto_rawDesc = "" + "\n" + "_signatureB\t\n" + "\a_signerB\x10\n" + - "\x0e_signingPubKey\"\x85\x02\n" + + "\x0e_signingPubKey\"\xc5\x01\n" + "\n" + "MFFilePath\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + "\x04size\x18\x02 \x01(\x03R\x04size\x12'\n" + - "\x06hashes\x18\x03 \x03(\v2\x0f.MFFileChecksumR\x06hashes\x12 \n" + - "\bmimeType\x18\xad\x02 \x01(\tH\x00R\bmimeType\x88\x01\x01\x12&\n" + + "\x06hashes\x18\x03 \x03(\v2\x0f.MFFileChecksumR\x06hashes\x12&\n" + "\x05mtime\x18\xae\x02 \x01(\v2\n" + - ".TimestampH\x01R\x05mtime\x88\x01\x01\x12&\n" + - "\x05ctime\x18\xaf\x02 \x01(\v2\n" + - ".TimestampH\x02R\x05ctime\x88\x01\x01\x12\x13\n" + - "\x04mode\x18\xb0\x02 \x01(\rR\x04modeB\v\n" + - "\t_mimeTypeB\b\n" + - "\x06_mtimeB\b\n" + - "\x06_ctime\".\n" + + ".TimestampH\x00R\x05mtime\x88\x01\x01\x12\x13\n" + + "\x04mode\x18\xb0\x02 \x01(\rR\x04modeB\b\n" + + "\x06_mtimeJ\x06\b\xad\x02\x10\xae\x02J\x06\b\xaf\x02\x10\xb0\x02R\bmimeTypeR\x05ctime\".\n" + "\x0eMFFileChecksum\x12\x1c\n" + "\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xd6\x01\n" + "\x06MFFile\x12)\n" + @@ -631,15 +610,14 @@ var file_mf_proto_depIdxs = []int32{ 1, // 1: MFFileOuter.compressionType:type_name -> MFFileOuter.CompressionType 6, // 2: MFFilePath.hashes:type_name -> MFFileChecksum 3, // 3: MFFilePath.mtime:type_name -> Timestamp - 3, // 4: MFFilePath.ctime:type_name -> Timestamp - 2, // 5: MFFile.version:type_name -> MFFile.Version - 5, // 6: MFFile.files:type_name -> MFFilePath - 3, // 7: MFFile.createdAt:type_name -> Timestamp - 8, // [8:8] is the sub-list for method output_type - 8, // [8:8] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name + 2, // 4: MFFile.version:type_name -> MFFile.Version + 5, // 5: MFFile.files:type_name -> MFFilePath + 3, // 6: MFFile.createdAt:type_name -> Timestamp + 7, // [7:7] is the sub-list for method output_type + 7, // [7:7] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_mf_proto_init() } diff --git a/mfer/mf.proto b/mfer/mf.proto index 3193e26..63f80c0 100644 --- a/mfer/mf.proto +++ b/mfer/mf.proto @@ -58,11 +58,13 @@ message MFFilePath { repeated MFFileChecksum hashes = 3; // optional per-file metadata - optional string mimeType = 301; optional Timestamp mtime = 302; - optional Timestamp ctime = 303; // permission bits, at most 0777; 0 when not recorded uint32 mode = 304; + + // used by no field, so that a later version cannot reuse them by accident + reserved 301, 303; + reserved "mimeType", "ctime"; } message MFFileChecksum { diff --git a/mfer/mf.proto.sha256 b/mfer/mf.proto.sha256 index 871641e..793f3fa 100644 --- a/mfer/mf.proto.sha256 +++ b/mfer/mf.proto.sha256 @@ -1 +1 @@ -1668b2f6b8639a9213873474e58dbacb3a4ef77cc27c42253cad8dd076e5f206 mf.proto +16a2e6b39986d02aa1c7c01dc6b97c9d4c30db7a9d000056bd5985a3fa7c8089 mf.proto diff --git a/mfer/mf_test.go b/mfer/mf_test.go index 83ef2d8..dbce16a 100644 --- a/mfer/mf_test.go +++ b/mfer/mf_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/reflect/protoreflect" "sneak.berlin/go/mfer/mfer" @@ -35,13 +36,11 @@ func TestFileEntryFieldsMatchSpec(t *testing.T) { t.Parallel() want := map[string]protoreflect.FieldNumber{ - "path": 1, - "size": 2, - "hashes": 3, - "mimeType": 301, - "mtime": 302, - "ctime": 303, - "mode": 304, + "path": 1, + "size": 2, + "hashes": 3, + "mtime": 302, + "mode": 304, } got := map[string]protoreflect.FieldNumber{} @@ -53,3 +52,16 @@ func TestFileEntryFieldsMatchSpec(t *testing.T) { require.Equal(t, want, got) } + +// A file entry reserves the numbers 301 and 303 and the names mimeType and +// ctime, as docs/FORMAT.md says, so that no field can take them. +func TestFileEntryReservesMIMETypeAndCtime(t *testing.T) { + t.Parallel() + + desc := (&mfer.MFFilePath{}).ProtoReflect().Descriptor() + + assert.True(t, desc.ReservedRanges().Has(301)) + assert.True(t, desc.ReservedRanges().Has(303)) + assert.True(t, desc.ReservedNames().Has("mimeType")) + assert.True(t, desc.ReservedNames().Has("ctime")) +} diff --git a/mfer/scanner.go b/mfer/scanner.go index 3c3db4e..fb6139a 100644 --- a/mfer/scanner.go +++ b/mfer/scanner.go @@ -76,7 +76,6 @@ type FileEntry struct { AbsPath AbsFilePath // Absolute path (used for reading file content) Size FileSize // File size in bytes Mtime ModTime // Last modification time - Ctime time.Time // Creation time (platform-dependent) Mode fs.FileMode // Permission bits (Perm() of the file's mode) } @@ -498,8 +497,6 @@ func (s *Scanner) enumerateFileWithInfo( Size: FileSize(info.Size()), Mtime: ModTime(info.ModTime()), Mode: info.Mode().Perm(), - // Note: Ctime not available from fs.FileInfo on all platforms - // Will need platform-specific code to extract it } s.mu.Lock()