2 Commits
Author SHA1 Message Date
sneak 977ceb1700 Load the manifest check downloads from memory, not a temp file (closes #223)
check / check (push) Waiting to run
check given a URL wrote the manifest it downloaded to a file in the temp
directory before loading it, so it failed wherever that directory is
missing or cannot be written, such as the scratch image script/cibuild
builds. It now keeps the manifest in memory, as fetch, list and export
do. NewChecker reads a manifest handed to it as a reader through the new
CheckerOptions.ManifestReader field, in place of ManifestPath, because it
otherwise reads the manifest from the filesystem holding the files it
checks. Only check uses that field. verifyRequiredSigner takes the
signer's fingerprint instead of a Checker, so fetch passes the signer of
the manifest it has already loaded and builds no Checker.

Model: opus-5-5
2026-10-15 19:28:31 +00:00
clawbot 7d601a5902 Require a SHA-256 hash in every manifest entry (closes #224)
check / check (push) Waiting to run
Loading accepted an entry whose hashes held no SHA-256 multihash, and
check and fetch, which compute only SHA-256, then reported the intact
file as a hash mismatch. Loading now refuses, naming the path, an entry
that does not list exactly one SHA-256 multihash; other hashes beside
it are allowed. AddFileWithHash accepts only a SHA-256 multihash, and
freshen carries an unchanged entry's SHA-256 hash over instead of its
first hash, found with SHA256Multihash, new in package mfer.
docs/FORMAT.md states the rule for writers and readers. Two loading
tests whose entries held an empty hash now give each entry a SHA-256
hash; the decoding-size test's path lengths change with it.

Model: opus-5-5
2026-10-15 21:25:27 +02:00
10 changed files with 268 additions and 174 deletions
+24 -15
View File
@@ -80,15 +80,15 @@ After decompressing `innerMessage`, the result is a serialized `MFFile`
Each file entry contains:
| Field | Number | Type | Description |
| ---------- | ------ | ------------------------- | ------------------------------------------------------------- |
| `path` | 1 | string | Relative file path (see Path Rules) |
| `size` | 2 | int64 | File size in bytes |
| `hashes` | 3 | repeated `MFFileChecksum` | At least one hash required; readers reject an entry with none |
| `mimeType` | 301 | string (optional) | MIME type |
| `mtime` | 302 | Timestamp (optional) | Modification time |
| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) |
| `mode` | 304 | uint32 | Permission bits (see File Mode) |
| Field | Number | Type | Description |
| ---------- | ------ | ------------------------- | ------------------------------------------------------------ |
| `path` | 1 | string | Relative file path (see Path Rules) |
| `size` | 2 | int64 | File size in bytes |
| `hashes` | 3 | repeated `MFFileChecksum` | Must include exactly one SHA-256 multihash (see Hash Format) |
| `mimeType` | 301 | string (optional) | MIME type |
| `mtime` | 302 | Timestamp (optional) | Modification time |
| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) |
| `mode` | 304 | uint32 | Permission bits (see File Mode) |
## File Mode
@@ -133,8 +133,16 @@ Each checksum is a single `bytes multiHash` field containing a
self-describing: the encoded bytes include a varint algorithm identifier
followed by a varint digest length followed by the digest itself.
The 1.0 implementation writes SHA-256 multihashes (`0x12` algorithm code).
Implementations must be able to verify SHA-256 multihashes at minimum.
A writer must give every file entry exactly one SHA-256 multihash of the file's
content: the algorithm code `0x12`, the digest length `0x20` and the 32 bytes of
the digest. It may list multihashes of other algorithms beside it. A reader must
refuse a manifest with an entry that has no SHA-256 multihash or more than one,
whatever other hashes the entry lists. A verifier then needs only SHA-256,
checking each file against its entry's SHA-256 multihash; it may ignore the
other hashes.
The reference implementation writes one hash for each entry, the SHA-256
multihash, and its `check` and `fetch` compute only SHA-256.
## Signature Scheme
@@ -212,10 +220,11 @@ compresses the same inner message to the same bytes:
- Each entry holds only `path`, `size`, `hashes` with exactly one hash, `mtime`
and `mode`. The hash is the SHA-256 multihash of the file's content, except
that `freshen` reads only the files that are new or whose size, `mtime` or
`mode` changed, and for every other file carries the first hash of its
existing entry over unchanged. `mtime` is the file's modification time as the
file system gives it, to the nanosecond, and `mode` is `0000` unless `gen` or
`freshen` is given `--include-permissions`.
`mode` changed, and for every other file carries the SHA-256 multihash of its
existing entry over unchanged, leaving out any other hash the entry lists.
`mtime` is the file's modification time as the file system gives it, to the
nanosecond, and `mode` is `0000` unless `gen` or `freshen` is given
`--include-permissions`.
- `createdAt` is left out unless `gen` or `freshen` is given
`--include-timestamps`. Then it holds the time the run started building the
manifest, to the nanosecond, so it differs on every run.
+6 -8
View File
@@ -121,8 +121,10 @@ func (mfa *CLIApp) downloadManifest(ctx context.Context, url string) ([]byte, er
}
// verifyRequiredSigner enforces the --require-signature fingerprint
// against the key that made the manifest's signature.
func verifyRequiredSigner(chk *mfer.Checker, requiredSigner string) error {
// against signer, the fingerprint of the key that made the manifest's
// signature, which is empty when the manifest is not signed. Loading the
// manifest checked that signer is that key's fingerprint.
func verifyRequiredSigner(signer, requiredSigner string) error {
// Validate fingerprint format: must be exactly 40 hex characters
if len(requiredSigner) != fingerprintHexLen {
return fmt.Errorf("%w, got %d", errInvalidFingerprint, len(requiredSigner))
@@ -133,15 +135,11 @@ func verifyRequiredSigner(chk *mfer.Checker, requiredSigner string) error {
return fmt.Errorf("invalid fingerprint: %w", err)
}
if !chk.IsSigned() {
if signer == "" {
return fmt.Errorf("%w, but signature from %s is required",
errManifestNotSigned, requiredSigner)
}
// Loading the manifest checked that the signer is the fingerprint of
// the key that made the signature.
signer := string(chk.Signer())
// Compare fingerprints - must be exact match (case-insensitive)
if !strings.EqualFold(signer, requiredSigner) {
return fmt.Errorf("embedded signing key fingerprint %s %w %s",
@@ -316,7 +314,7 @@ func (mfa *CLIApp) checkManifestOperation(
// Check signature requirement. An empty fingerprint is checked too,
// and refused as one of the wrong length.
if cmd.IsSet(flagRequireSignature) {
err = verifyRequiredSigner(chk, cmd.String(flagRequireSignature))
err = verifyRequiredSigner(string(chk.Signer()), cmd.String(flagRequireSignature))
if err != nil {
return err
}
+6 -38
View File
@@ -50,38 +50,6 @@ func runLocked(fn func() error) error {
return fn()
}
// unsignedChecker builds a Checker over a freshly scanned, unsigned manifest.
func unsignedChecker(t *testing.T) *mfer.Checker {
t.Helper()
// The scan and the manifest load log to the process-global logger, so
// serialize with the CLI runs.
runMu.Lock()
defer runMu.Unlock()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/d", 0o755))
require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
s := mfer.NewScannerWithOptions(&mfer.ScannerOptions{Fs: fs})
require.NoError(t, s.EnumeratePath("/d", nil))
var buf bytes.Buffer
require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644))
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestPath: "/d/index.mf",
BasePath: "/d",
Fs: fs,
})
require.NoError(t, err)
require.False(t, chk.IsSigned())
return chk
}
func TestNoManifestFoundMessage(t *testing.T) {
t.Parallel()
@@ -97,7 +65,7 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
t.Run("invalid fingerprint length", func(t *testing.T) {
t.Parallel()
err := verifyRequiredSigner(unsignedChecker(t), "12345678")
err := verifyRequiredSigner("", "12345678")
require.ErrorIs(t, err, errInvalidFingerprint)
assert.EqualError(t, err,
"invalid fingerprint: must be exactly 40 hex characters, got 8")
@@ -106,7 +74,7 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
t.Run("manifest not signed", func(t *testing.T) {
t.Parallel()
err := verifyRequiredSigner(unsignedChecker(t), msgFpA)
err := verifyRequiredSigner("", msgFpA)
require.ErrorIs(t, err, errManifestNotSigned)
assert.EqualError(t, err,
"manifest is not signed, but signature from "+msgFpA+" is required")
@@ -121,13 +89,13 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
func TestSignerMismatchMessage(t *testing.T) {
t.Parallel()
chk := signedChecker(t,
signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")}))
signer := string(signedChecker(t,
signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")})).Signer())
err := verifyRequiredSigner(chk, msgFpB)
err := verifyRequiredSigner(signer, msgFpB)
require.ErrorIs(t, err, errSignerMismatch)
assert.EqualError(t, err,
"embedded signing key fingerprint "+string(chk.Signer())+
"embedded signing key fingerprint "+signer+
" does not match required "+msgFpB)
}
+1 -16
View File
@@ -474,7 +474,7 @@ func (mfa *CLIApp) fetchManifest(
}
if requiredSigner != "" {
err = verifyFetchedSigner(manifestData, requiredSigner)
err = verifyRequiredSigner(string(manifest.Signer()), requiredSigner)
if err != nil {
return nil, nil, err
}
@@ -552,21 +552,6 @@ func checkNoNameClash(files []*mfer.MFFilePath) error {
return nil
}
// verifyFetchedSigner enforces --require-signature on the fetched manifest
// exactly as check does. The Checker it builds is used only for the
// manifest's signature, so its base path is never read.
func verifyFetchedSigner(manifestData []byte, requiredSigner string) error {
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestReader: bytes.NewReader(manifestData),
BasePath: "/",
})
if err != nil {
return fmt.Errorf("load manifest: %w", err)
}
return verifyRequiredSigner(chk, requiredSigner)
}
// saveManifest writes the fetched manifest into dest under the default
// manifest name, the way fetch writes every file: to a new temp file that
// is then renamed into place.
+11 -11
View File
@@ -33,9 +33,9 @@ var (
// modification time where one is required to carry it forward unchanged.
errEntryMissingMtime = errors.New("manifest entry has no mtime")
// errEntryMissingHash indicates a manifest entry that carries no hash
// to carry forward.
errEntryMissingHash = errors.New("manifest entry has no hash")
// errEntryMissingHash indicates a manifest entry that carries no
// SHA-256 hash to carry forward.
errEntryMissingHash = errors.New("manifest entry has no SHA-256 hash")
)
// errSignKeyNeeded indicates a signed manifest given to freshen with no
@@ -699,26 +699,26 @@ func addFileToBuilder(
// Entries reach this path only when recordEntry classified them as
// unchanged, which requires a recorded mtime, so an absent mtime here is
// an error rather than something to paper over with the Unix epoch.
// Loading refuses an entry with no hash; one that got here anyway is an
// error too, not a file to leave out of the manifest.
// Only the entry's SHA-256 hash is carried over, wherever the entry lists
// it, since check and fetch compare with no other. Loading refuses an
// entry with no SHA-256 hash; one that got here anyway is an error too,
// not a file to leave out of the manifest.
func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
mtime, ok := entryMtime(entry)
if !ok {
return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath())
}
if len(entry.GetHashes()) == 0 {
hash := mfer.SHA256Multihash(entry)
if hash == nil {
return fmt.Errorf("%w: %s", errEntryMissingHash, entry.GetPath())
}
err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
fs.FileMode(entry.GetMode()), entry.GetHashes()[0].GetMultiHash())
fs.FileMode(entry.GetMode()), hash)
if err != nil {
return fmt.Errorf(
"manifest entry %s: %w (regenerate the manifest with mfer generate)",
entry.GetPath(), err,
)
return fmt.Errorf("manifest entry %s: %w", entry.GetPath(), err)
}
return nil
+60 -29
View File
@@ -2,7 +2,10 @@
package cli
import (
"bytes"
"context"
"crypto/sha256"
"crypto/sha512"
"maps"
"os"
"path/filepath"
@@ -584,48 +587,76 @@ func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
assert.Contains(t, err.Error(), "file1.txt")
}
// TestFreshenAddExistingRejectsMissingHash pins that an entry with no hash
// is an error rather than a file left out of the rebuilt manifest.
// TestFreshenAddExistingRejectsMissingHash pins that an entry with no
// SHA-256 hash, whether it lists no hash or only a SHA-1 one, is an error
// rather than a file left out of the rebuilt manifest.
func TestFreshenAddExistingRejectsMissingHash(t *testing.T) {
t.Parallel()
b := mfer.NewBuilder()
entry := &mfer.MFFilePath{
Path: "data.bin",
Size: 8,
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
}
err := addExistingToBuilder(b, entry)
require.ErrorIs(t, err, errEntryMissingHash)
assert.Contains(t, err.Error(), "data.bin")
assert.Zero(t, b.FileCount())
}
// TestFreshenAddExistingRejectsShortHash pins that an existing manifest
// entry whose hash the builder refuses is reported as a problem with the
// manifest, with the command that fixes it.
func TestFreshenAddExistingRejectsShortHash(t *testing.T) {
t.Parallel()
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
require.NoError(t, err)
tests := []struct {
name string
hashes []*mfer.MFFileChecksum
}{
{"no hash", nil},
{"SHA-1 hash only", []*mfer.MFFileChecksum{{MultiHash: sha1Hash}}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
b := mfer.NewBuilder()
entry := &mfer.MFFilePath{
Path: "data.bin",
Size: 8,
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
Hashes: tt.hashes,
}
err := addExistingToBuilder(b, entry)
require.ErrorIs(t, err, errEntryMissingHash)
assert.Contains(t, err.Error(), "data.bin")
assert.Zero(t, b.FileCount())
})
}
}
// TestFreshenAddExistingKeepsSHA256Hash pins that an unchanged entry that
// lists another hash before its SHA-256 hash is carried over with the
// SHA-256 hash, the one check and fetch compare with, and nothing else.
func TestFreshenAddExistingKeepsSHA256Hash(t *testing.T) {
t.Parallel()
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
require.NoError(t, err)
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
b := mfer.NewBuilder()
entry := &mfer.MFFilePath{
Path: "old.txt",
Size: 8,
Path: "kept.txt",
Size: 6,
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
Hashes: []*mfer.MFFileChecksum{
{MultiHash: sha1Hash},
{MultiHash: sha512Hash},
{MultiHash: sha256Hash},
},
}
err = addExistingToBuilder(b, entry)
require.Error(t, err)
assert.Contains(t, err.Error(), "manifest entry old.txt")
assert.Contains(t, err.Error(), "mfer generate")
assert.Zero(t, b.FileCount())
require.NoError(t, addExistingToBuilder(b, entry))
var buf bytes.Buffer
require.NoError(t, b.Build(context.Background(), &buf))
m, err := mfer.NewManifestFromReader(&buf)
require.NoError(t, err)
require.Len(t, m.Files(), 1)
require.Len(t, m.Files()[0].GetHashes(), 1)
assert.Equal(t, sha256Hash, m.Files()[0].GetHashes()[0].GetMultiHash())
}
// TestEntryMtime pins the presence semantics the callers depend on.
+26 -17
View File
@@ -37,8 +37,7 @@ var (
errPathDot = errors.New("contains '.' segment")
errSizeMismatch = errors.New("size mismatch")
errNegativeSize = errors.New("size cannot be negative")
errHashNotMultihash = errors.New("hash is not a valid multihash")
errHashTooShort = errors.New("hash digest is too short")
errHashNotSHA256 = errors.New("hash is not a SHA-256 multihash")
errDuplicatePath = errors.New("duplicate path")
)
@@ -113,6 +112,28 @@ func (m ModTime) Timestamp() *Timestamp {
// Multihash represents a multihash-encoded file hash (typically SHA2-256).
type Multihash []byte
// SHA256Multihash returns the first of entry's hashes that is a SHA-256
// multihash, or nil if none is. Loading a manifest refuses an entry that
// does not list exactly one, so for a loaded entry it is the only one.
func SHA256Multihash(entry *MFFilePath) Multihash {
for _, h := range entry.GetHashes() {
if isSHA256Multihash(h.GetMultiHash()) {
return h.GetMultiHash()
}
}
return nil
}
// isSHA256Multihash reports whether hash is a SHA-256 multihash: the
// multihash code for SHA-256, then a 32-byte digest.
func isSHA256Multihash(hash []byte) bool {
decoded, err := multihash.Decode(hash)
return err == nil && decoded.Code == multihash.SHA2_256 &&
len(decoded.Digest) == sha256.Size
}
// FileHashProgress reports progress during file hashing.
type FileHashProgress struct {
BytesRead FileSize // Total bytes read so far for the current file
@@ -241,8 +262,7 @@ func (b *Builder) FileCount() int {
// This is useful when the hash is already known (e.g., from an existing manifest).
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
// Returns an error if path is invalid or already added, size is negative,
// or hash is not a multihash with a digest of at least 32 bytes, as long
// as SHA-256's.
// or hash is not a SHA-256 multihash, the hash every entry must carry.
func (b *Builder) AddFileWithHash(
path RelFilePath,
size FileSize,
@@ -259,19 +279,8 @@ func (b *Builder) AddFileWithHash(
return errNegativeSize
}
decoded, err := multihash.Decode(hash)
if err != nil {
return fmt.Errorf("%w: %w", errHashNotMultihash, err)
}
// The reader's limit on decoding cost (maxDecodedGrowth) assumes every
// hash is at least as long as a SHA-256 multihash, so a manifest of
// shorter ones could fail to load.
if len(decoded.Digest) < sha256.Size {
return fmt.Errorf(
"%w: %d bytes, at least %d needed",
errHashTooShort, len(decoded.Digest), sha256.Size,
)
if !isSHA256Multihash(hash) {
return errHashNotSHA256
}
entry := &MFFilePath{
+13 -10
View File
@@ -5,6 +5,7 @@ import (
"bytes"
"context"
"crypto/sha256"
"crypto/sha512"
"fmt"
"path/filepath"
"strings"
@@ -94,23 +95,25 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
require.NoError(t, err)
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
require.NoError(t, err)
truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
require.NoError(t, err)
tests := []struct {
name string
hash Multihash
want error
}{
{"nil hash", nil, errHashNotMultihash},
{"empty hash", []byte{}, errHashNotMultihash},
{"one-byte hash", []byte{0x12}, errHashNotMultihash},
{"nil hash", nil},
{"empty hash", []byte{}},
{"one-byte hash", []byte{0x12}},
// A SHA-256 code and 32-byte length, then only two bytes of digest.
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
// A valid multihash, but its 20-byte SHA-1 digest is too short.
{"SHA-1 multihash", sha1Hash, errHashTooShort},
// A valid multihash whose 31-byte digest is one byte short.
{"31-byte digest", truncatedHash, errHashTooShort},
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}},
{"SHA-1 multihash", sha1Hash},
{"SHA-512 multihash", sha512Hash},
// A SHA-256 code with a 31-byte digest.
{"31-byte digest", truncatedHash},
}
for _, tt := range tests {
@@ -119,7 +122,7 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
b := NewBuilder()
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), 0, tt.hash)
require.ErrorIs(t, err, tt.want)
require.ErrorIs(t, err, errHashNotSHA256)
assert.Equal(t, 0, b.FileCount())
})
}
+31 -5
View File
@@ -27,7 +27,8 @@ var (
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
errInvalidFileFormat = errors.New("invalid file format")
errInvalidManifestPath = errors.New("invalid file entry")
errEntryMissingHash = errors.New("file entry has no hash")
errEntryMissingHash = errors.New("file entry has no SHA-256 hash")
errEntryExtraSHA256Hash = errors.New("file entry has more than one SHA-256 hash")
errDecodedTooLarge = errors.New("too much memory needed")
errSignerNotSigningKey = errors.New(
"signer is not the fingerprint of the key that made the signature")
@@ -236,6 +237,30 @@ func forEachBytesField(
return nil
}
// validateEntryHashes refuses, naming its path, an entry that does not list
// exactly one SHA-256 multihash; hashes of other algorithms beside it are
// allowed. check and fetch compute only SHA-256, so with none they would
// have nothing to check the file against, and with two check would pass a
// file that matches either.
func validateEntryHashes(entry *MFFilePath) error {
sha256Hashes := 0
for _, h := range entry.GetHashes() {
if isSHA256Multihash(h.GetMultiHash()) {
sha256Hashes++
}
}
switch {
case sha256Hashes == 0:
return fmt.Errorf("%w: %q", errEntryMissingHash, entry.GetPath())
case sha256Hashes > 1:
return fmt.Errorf("%w: %q", errEntryExtraSHA256Hash, entry.GetPath())
default:
return nil
}
}
func (m *manifest) deserializeInner() error {
err := m.validateOuterHeader()
if err != nil {
@@ -294,8 +319,8 @@ func (m *manifest) deserializeInner() error {
// untrusted .mf. Reject loudly on the first offender rather than
// dropping entries, which would let a hostile manifest hide files from a
// check. A path listed twice is refused too: check would check the one
// file against both entries. So is an entry with no hash: there is
// nothing to check the file against.
// file against both entries. So is an entry that does not list exactly
// one SHA-256 hash.
seen := make(map[string]bool, len(m.pbInner.GetFiles()))
for _, f := range m.pbInner.GetFiles() {
@@ -304,8 +329,9 @@ func (m *manifest) deserializeInner() error {
return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
}
if len(f.GetHashes()) == 0 {
return fmt.Errorf("%w: %q", errEntryMissingHash, f.GetPath())
err = validateEntryHashes(f)
if err != nil {
return err
}
if seen[f.GetPath()] {
+90 -25
View File
@@ -5,8 +5,10 @@ import (
"bytes"
"context"
"crypto/sha256"
"crypto/sha512"
"fmt"
"strconv"
"strings"
"testing"
"time"
"uuid"
@@ -167,40 +169,94 @@ func TestDeserializeRefusesPathListedTwice(t *testing.T) {
}
}
// A manifest holding a file entry with no hash is refused as it is loaded,
// naming the path: there is nothing to check the file against.
func TestDeserializeRefusesEntryWithNoHash(t *testing.T) {
// A file entry must list exactly one SHA-256 hash, and may list hashes of
// other algorithms beside it. An entry with none, even one that lists a
// SHA-512 hash, is refused as it is loaded, and so is an entry with two;
// each refusal names the path. An entry listing a SHA-512 hash and then a
// SHA-256 one loads with both.
func TestDeserializeRequiresExactlyOneSHA256Hash(t *testing.T) {
t.Parallel()
id := uuid.NewV4()
inner, err := proto.Marshal(&MFFile{
Version: MFFile_VERSION_ONE,
Files: []*MFFilePath{{Path: "x", Size: 1}},
Uuid: id[:],
})
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
_, err = NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
require.ErrorIs(t, err, errEntryMissingHash)
require.EqualError(t, err, `file entry has no hash: "x"`)
otherSHA256Hash, err := multihash.Encode(
bytes.Repeat([]byte{1}, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
require.NoError(t, err)
tests := []struct {
name string
hashes [][]byte
want error // nil for an entry that loads
}{
{"no hash", nil, errEntryMissingHash},
{"SHA-512 hash only", [][]byte{sha512Hash}, errEntryMissingHash},
{
"two SHA-256 hashes",
[][]byte{sha256Hash, otherSHA256Hash},
errEntryExtraSHA256Hash,
},
{"SHA-512 hash then SHA-256 hash", [][]byte{sha512Hash, sha256Hash}, nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
entry := &MFFilePath{Path: "x", Size: 1}
for _, h := range tt.hashes {
entry.Hashes = append(entry.Hashes, &MFFileChecksum{MultiHash: h})
}
id := uuid.NewV4()
inner, err := proto.Marshal(&MFFile{
Version: MFFile_VERSION_ONE,
Files: []*MFFilePath{entry},
Uuid: id[:],
})
require.NoError(t, err)
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
if tt.want == nil {
require.NoError(t, err)
require.Len(t, m.Files(), 1)
assert.Len(t, m.Files()[0].GetHashes(), len(tt.hashes))
return
}
require.ErrorIs(t, err, tt.want)
assert.Contains(t, err.Error(), `"x"`)
})
}
}
// Entries of a path, an empty hash, an empty MIME type and empty modification
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
// and take 16 bytes plus the path to encode. A 37-character path makes that
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
// it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
// padded with zeros to that length, since a manifest lists a path only once.
// Entries of a path, a SHA-256 hash, an empty MIME type and empty
// modification and change times are counted at 432 bytes each (176 + 112 +
// 16 + 64 + 64) and take 52 bytes plus the path to encode. A one-character
// path makes that 53 bytes, about 8.2 times: refused, and leaving any one of
// the five uncounted, even the MIME type, brings it under 8. A two-character
// path makes it 54 bytes, 8 times exactly: loaded. There is one entry for
// each base-36 digit, and each entry's path is its number in base 36, padded
// with zeros to that length, since a manifest lists a path only once.
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
t.Parallel()
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
require.NoError(t, err)
tests := []struct {
pathLen int
refused bool
}{
{37, true},
{39, false},
{1, true},
{2, false},
}
for _, tt := range tests {
@@ -213,11 +269,14 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
inner = protowire.AppendBytes(inner, id[:])
for i := range 1000 {
for i := range 36 {
path := strconv.FormatInt(int64(i), 36)
path = strings.Repeat("0", tt.pathLen-len(path)) + path
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
entry = protowire.AppendString(entry, path)
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
entry = protowire.AppendBytes(entry, nil)
entry = protowire.AppendBytes(entry, checksum)
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
entry = protowire.AppendBytes(entry, nil)
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
@@ -247,10 +306,16 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
unknown = protowire.AppendBytes(unknown, []byte("not known"))
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
require.NoError(t, err)
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
entry = protowire.AppendString(entry, "a")
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
entry = protowire.AppendBytes(entry, nil)
entry = protowire.AppendBytes(entry, checksum)
entry = append(entry, unknown...)
id := uuid.NewV4()