Load the manifest check downloads from memory, not a temp file (closes #223)

check given a URL wrote the manifest it downloaded to a file in the temp
directory before loading it, so it failed wherever that directory is
missing or cannot be written, such as the scratch image script/cibuild
builds. It now keeps the manifest in memory, as fetch, list and export
do. NewChecker reads a manifest handed to it as a reader through the new
CheckerOptions.ManifestReader field, in place of ManifestPath, because it
otherwise reads the manifest from the filesystem holding the files it
checks. Only check uses that field. verifyRequiredSigner takes the
signer's fingerprint instead of a Checker, so fetch passes the signer of
the manifest it has already loaded and builds no Checker.

Model: opus-5-5

docs/FORMAT.md

@@ -80,15 +80,15 @@ After decompressing `innerMessage`, the result is a serialized `MFFile`
 
 Each file entry contains:
 
-| Field      | Number | Type                      | Description                                                   |
+| Field      | Number | Type                      | Description                                                  |
-| ---------- | ------ | ------------------------- | ------------------------------------------------------------- |
+| ---------- | ------ | ------------------------- | ------------------------------------------------------------ |
-| `path`     | 1      | string                    | Relative file path (see Path Rules)                           |
+| `path`     | 1      | string                    | Relative file path (see Path Rules)                          |
-| `size`     | 2      | int64                     | File size in bytes                                            |
+| `size`     | 2      | int64                     | File size in bytes                                           |
-| `hashes`   | 3      | repeated `MFFileChecksum` | At least one hash required; readers reject an entry with none |
+| `hashes`   | 3      | repeated `MFFileChecksum` | Must include exactly one SHA-256 multihash (see Hash Format) |
-| `mimeType` | 301    | string (optional)         | MIME type                                                     |
+| `mimeType` | 301    | string (optional)         | MIME type                                                    |
-| `mtime`    | 302    | Timestamp (optional)      | Modification time                                             |
+| `mtime`    | 302    | Timestamp (optional)      | Modification time                                            |
-| `ctime`    | 303    | Timestamp (optional)      | Change time (inode metadata change)                           |
+| `ctime`    | 303    | Timestamp (optional)      | Change time (inode metadata change)                          |
-| `mode`     | 304    | uint32                    | Permission bits (see File Mode)                               |
+| `mode`     | 304    | uint32                    | Permission bits (see File Mode)                              |
 
 ## File Mode
 
@@ -133,8 +133,16 @@ Each checksum is a single `bytes multiHash` field containing a
 self-describing: the encoded bytes include a varint algorithm identifier
 followed by a varint digest length followed by the digest itself.
 
-The 1.0 implementation writes SHA-256 multihashes (`0x12` algorithm code).
+A writer must give every file entry exactly one SHA-256 multihash of the file's
-Implementations must be able to verify SHA-256 multihashes at minimum.
+content: the algorithm code `0x12`, the digest length `0x20` and the 32 bytes of
+the digest. It may list multihashes of other algorithms beside it. A reader must
+refuse a manifest with an entry that has no SHA-256 multihash or more than one,
+whatever other hashes the entry lists. A verifier then needs only SHA-256,
+checking each file against its entry's SHA-256 multihash; it may ignore the
+other hashes.
+
+The reference implementation writes one hash for each entry, the SHA-256
+multihash, and its `check` and `fetch` compute only SHA-256.
 
 ## Signature Scheme
 
@@ -212,10 +220,11 @@ compresses the same inner message to the same bytes:
 - Each entry holds only `path`, `size`, `hashes` with exactly one hash, `mtime`
   and `mode`. The hash is the SHA-256 multihash of the file's content, except
   that `freshen` reads only the files that are new or whose size, `mtime` or
-  `mode` changed, and for every other file carries the first hash of its
+  `mode` changed, and for every other file carries the SHA-256 multihash of its
-  existing entry over unchanged. `mtime` is the file's modification time as the
+  existing entry over unchanged, leaving out any other hash the entry lists.
-  file system gives it, to the nanosecond, and `mode` is `0000` unless `gen` or
+  `mtime` is the file's modification time as the file system gives it, to the
-  `freshen` is given `--include-permissions`.
+  nanosecond, and `mode` is `0000` unless `gen` or `freshen` is given
+  `--include-permissions`.
 - `createdAt` is left out unless `gen` or `freshen` is given
   `--include-timestamps`. Then it holds the time the run started building the
   manifest, to the nanosecond, so it differs on every run.

internal/cli/check.go

@@ -2,6 +2,7 @@
 package cli
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"errors"
@@ -94,47 +95,36 @@ func findManifest(fs afero.Fs, dir string) (string, error) {
 	return path, nil
 }
 
-// fetchManifestToTemp downloads a manifest URL to a temporary file and
+// downloadManifest downloads the manifest at url into memory. It refuses a
-// returns the temp file path. The caller is responsible for removing it.
+// manifest larger than mfa.maxManifestSize.
-func (mfa *CLIApp) fetchManifestToTemp(
+func (mfa *CLIApp) downloadManifest(ctx context.Context, url string) ([]byte, error) {
-	ctx context.Context, url string,
+	rc, err := mfa.openManifestReader(ctx, url)
-) (string, error) {
+	if err != nil {
-	rc, fetchErr := mfa.openManifestReader(ctx, url)
+		return nil, err
-	if fetchErr != nil {
-		return "", fetchErr
 	}
 
-	tmpFile, tmpErr := afero.TempFile(mfa.Fs, "", "mfer-manifest-*.mf")
+	defer func() { _ = rc.Close() }()
-	if tmpErr != nil {
-		_ = rc.Close()
 
-		return "", tmpErr
+	// Reading stops one byte past mfa.maxManifestSize, which is enough to
-	}
-
-	tmpPath := tmpFile.Name()
-
-	// Copying stops one byte past mfa.maxManifestSize, which is enough to
 	// tell that the manifest is too large.
-	written, cpErr := io.Copy(tmpFile, io.LimitReader(rc, mfa.maxManifestSize+1))
+	manifestData, err := io.ReadAll(io.LimitReader(rc, mfa.maxManifestSize+1))
-	_ = rc.Close()
+	if err != nil {
-	_ = tmpFile.Close()
+		return nil, fmt.Errorf("download manifest: %w", err)
-
-	if cpErr == nil && written > mfa.maxManifestSize {
-		cpErr = fmt.Errorf("%w of %d bytes", errManifestTooLarge, mfa.maxManifestSize)
 	}
 
-	if cpErr != nil {
+	if int64(len(manifestData)) > mfa.maxManifestSize {
-		_ = mfa.Fs.Remove(tmpPath)
+		return nil, fmt.Errorf("download manifest: %w of %d bytes",
-
+			errManifestTooLarge, mfa.maxManifestSize)
-		return "", fmt.Errorf("download manifest: %w", cpErr)
 	}
 
-	return tmpPath, nil
+	return manifestData, nil
 }
 
 // verifyRequiredSigner enforces the --require-signature fingerprint
-// against the key that made the manifest's signature.
+// against signer, the fingerprint of the key that made the manifest's
-func verifyRequiredSigner(chk *mfer.Checker, requiredSigner string) error {
+// signature, which is empty when the manifest is not signed. Loading the
+// manifest checked that signer is that key's fingerprint.
+func verifyRequiredSigner(signer, requiredSigner string) error {
 	// Validate fingerprint format: must be exactly 40 hex characters
 	if len(requiredSigner) != fingerprintHexLen {
 		return fmt.Errorf("%w, got %d", errInvalidFingerprint, len(requiredSigner))
@@ -145,15 +135,11 @@ func verifyRequiredSigner(chk *mfer.Checker, requiredSigner string) error {
 		return fmt.Errorf("invalid fingerprint: %w", err)
 	}
 
-	if !chk.IsSigned() {
+	if signer == "" {
 		return fmt.Errorf("%w, but signature from %s is required",
 			errManifestNotSigned, requiredSigner)
 	}
 
-	// Loading the manifest checked that the signer is the fingerprint of
-	// the key that made the signature.
-	signer := string(chk.Signer())
-
 	// Compare fingerprints - must be exact match (case-insensitive)
 	if !strings.EqualFold(signer, requiredSigner) {
 		return fmt.Errorf("embedded signing key fingerprint %s %w %s",
@@ -299,32 +285,28 @@ func (mfa *CLIApp) checkManifestOperation(
 		return err
 	}
 
-	// Done before a URL is swapped for the temp file it is downloaded to,
-	// whose directory is not the base.
 	basePath := resolveBasePath(cmd, manifestPath)
 
-	// URL manifests need to be downloaded to a temp file for the checker
+	opts := &mfer.CheckerOptions{BasePath: basePath, Fs: mfa.Fs}
+
+	// A manifest given by URL is downloaded into memory and read from
+	// there.
 	if isHTTPURL(manifestPath) {
-		tmpPath, tmpErr := mfa.fetchManifestToTemp(ctx, manifestPath)
+		manifestData, downloadErr := mfa.downloadManifest(ctx, manifestPath)
-		if tmpErr != nil {
+		if downloadErr != nil {
-			return tmpErr
+			return downloadErr
 		}
 
-		defer func() { _ = mfa.Fs.Remove(tmpPath) }()
+		opts.ManifestReader = bytes.NewReader(manifestData)
-
+	} else {
-		manifestPath = tmpPath
+		opts.ManifestPath = manifestPath
 	}
 
 	showProgress := cmd.Bool("progress")
 
 	log.Infof("checking manifest %s with base %s", manifestPath, basePath)
 
-	// Create checker
+	chk, err := mfer.NewChecker(opts)
-	chk, err := mfer.NewChecker(&mfer.CheckerOptions{
-		ManifestPath: manifestPath,
-		BasePath:     basePath,
-		Fs:           mfa.Fs,
-	})
 	if err != nil {
 		return fmt.Errorf("load manifest: %w", err)
 	}
@@ -332,7 +314,7 @@ func (mfa *CLIApp) checkManifestOperation(
 	// Check signature requirement. An empty fingerprint is checked too,
 	// and refused as one of the wrong length.
 	if cmd.IsSet(flagRequireSignature) {
-		err = verifyRequiredSigner(chk, cmd.String(flagRequireSignature))
+		err = verifyRequiredSigner(string(chk.Signer()), cmd.String(flagRequireSignature))
 		if err != nil {
 			return err
 		}

internal/cli/entry_test.go

@@ -1167,6 +1167,29 @@ func TestCheckURLManifestResolvesEntriesAgainstCurrentDirectory(t *testing.T) {
 	assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
 }
 
+// TestCheckURLManifestWithoutTempDirectory checks the tree under --base
+// against a manifest given by URL while TMPDIR names a directory that does
+// not exist. check keeps the manifest it downloads in memory, so it passes.
+// Setting TMPDIR keeps it from running in parallel with other tests.
+func TestCheckURLManifestWithoutTempDirectory(t *testing.T) {
+	files := map[string][]byte{testFileTxt: []byte("hello")}
+
+	server := httptest.NewServer(fetchTestHandler(manifestOf(t, files), files))
+	defer server.Close()
+
+	base := t.TempDir()
+	require.NoError(t,
+		os.WriteFile(filepath.Join(base, testFileTxt), files[testFileTxt], 0o600))
+
+	t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing"))
+
+	opts := testOpts([]string{
+		testApp, cmdCheck, testFlagNoExtra, testFlagBase, base,
+		server.URL + "/" + defaultManifestName,
+	}, afero.NewOsFs())
+	assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+}
+
 // unlistableDirFs is a filesystem on which one directory cannot be listed.
 type unlistableDirFs struct {
 	afero.Fs

internal/cli/errmsg_test.go

@@ -50,38 +50,6 @@ func runLocked(fn func() error) error {
 	return fn()
 }
 
-// unsignedChecker builds a Checker over a freshly scanned, unsigned manifest.
-func unsignedChecker(t *testing.T) *mfer.Checker {
-	t.Helper()
-
-	// The scan and the manifest load log to the process-global logger, so
-	// serialize with the CLI runs.
-	runMu.Lock()
-	defer runMu.Unlock()
-
-	fs := afero.NewMemMapFs()
-	require.NoError(t, fs.MkdirAll("/d", 0o755))
-	require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
-
-	s := mfer.NewScannerWithOptions(&mfer.ScannerOptions{Fs: fs})
-	require.NoError(t, s.EnumeratePath("/d", nil))
-
-	var buf bytes.Buffer
-
-	require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
-	require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644))
-
-	chk, err := mfer.NewChecker(&mfer.CheckerOptions{
-		ManifestPath: "/d/index.mf",
-		BasePath:     "/d",
-		Fs:           fs,
-	})
-	require.NoError(t, err)
-	require.False(t, chk.IsSigned())
-
-	return chk
-}
-
 func TestNoManifestFoundMessage(t *testing.T) {
 	t.Parallel()
 
@@ -97,7 +65,7 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
 	t.Run("invalid fingerprint length", func(t *testing.T) {
 		t.Parallel()
 
-		err := verifyRequiredSigner(unsignedChecker(t), "12345678")
+		err := verifyRequiredSigner("", "12345678")
 		require.ErrorIs(t, err, errInvalidFingerprint)
 		assert.EqualError(t, err,
 			"invalid fingerprint: must be exactly 40 hex characters, got 8")
@@ -106,7 +74,7 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
 	t.Run("manifest not signed", func(t *testing.T) {
 		t.Parallel()
 
-		err := verifyRequiredSigner(unsignedChecker(t), msgFpA)
+		err := verifyRequiredSigner("", msgFpA)
 		require.ErrorIs(t, err, errManifestNotSigned)
 		assert.EqualError(t, err,
 			"manifest is not signed, but signature from "+msgFpA+" is required")
@@ -121,13 +89,13 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
 func TestSignerMismatchMessage(t *testing.T) {
 	t.Parallel()
 
-	chk := signedChecker(t,
+	signer := string(signedChecker(t,
-		signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")}))
+		signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")})).Signer())
 
-	err := verifyRequiredSigner(chk, msgFpB)
+	err := verifyRequiredSigner(signer, msgFpB)
 	require.ErrorIs(t, err, errSignerMismatch)
 	assert.EqualError(t, err,
-		"embedded signing key fingerprint "+string(chk.Signer())+
+		"embedded signing key fingerprint "+signer+
 			" does not match required "+msgFpB)
 }
 

internal/cli/fetch.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/multiformats/go-multihash"
-	"github.com/spf13/afero"
 	"github.com/urfave/cli/v3"
 	"sneak.berlin/go/mfer/internal/log"
 	"sneak.berlin/go/mfer/mfer"
@@ -475,7 +474,7 @@ func (mfa *CLIApp) fetchManifest(
 	}
 
 	if requiredSigner != "" {
-		err = verifyFetchedSigner(manifestData, requiredSigner)
+		err = verifyRequiredSigner(string(manifest.Signer()), requiredSigner)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -553,31 +552,6 @@ func checkNoNameClash(files []*mfer.MFFilePath) error {
 	return nil
 }
 
-// verifyFetchedSigner enforces --require-signature on the fetched manifest
-// exactly as check does. verifyRequiredSigner takes a Checker, which loads
-// its manifest from a file, so the manifest is handed to it as a file in
-// memory.
-func verifyFetchedSigner(manifestData []byte, requiredSigner string) error {
-	memFs := afero.NewMemMapFs()
-	manifestPath := "/" + defaultManifestName
-
-	err := afero.WriteFile(memFs, manifestPath, manifestData, filePerms)
-	if err != nil {
-		return err
-	}
-
-	chk, err := mfer.NewChecker(&mfer.CheckerOptions{
-		ManifestPath: manifestPath,
-		BasePath:     "/",
-		Fs:           memFs,
-	})
-	if err != nil {
-		return fmt.Errorf("load manifest: %w", err)
-	}
-
-	return verifyRequiredSigner(chk, requiredSigner)
-}
-
 // saveManifest writes the fetched manifest into dest under the default
 // manifest name, the way fetch writes every file: to a new temp file that
 // is then renamed into place.

internal/cli/fetch_test.go

@@ -489,18 +489,17 @@ func TestFetchStopsReadingFilePastListedSize(t *testing.T) {
 
 // TestManifestDownloadStopsAtLimit serves a manifest that never ends to
 // fetch and to check, with the most they download of a manifest lowered
-// to 64 KiB. Each must stop reading at that limit, fail with an error
+// to 64 KiB. Each must stop reading at that limit and fail with an error
-// naming it and leave no temp file.
+// naming it.
 func TestManifestDownloadStopsAtLimit(t *testing.T) {
+	t.Parallel()
+
 	server := httptest.NewServer(
 		http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			_, _ = io.Copy(w, zeros{})
 		}))
 	defer server.Close()
 
-	tmpDir := t.TempDir()
-	t.Setenv("TMPDIR", tmpDir)
-
 	mfa := &CLIApp{Fs: afero.NewOsFs(), maxManifestSize: 64 << 10}
 
 	fetch := mfa.fetchCommand()
@@ -520,10 +519,6 @@ func TestManifestDownloadStopsAtLimit(t *testing.T) {
 			"download manifest: file exceeds maximum allowed size of 65536 bytes",
 			cmd.Name)
 	}
-
-	leftover, err := os.ReadDir(tmpDir)
-	require.NoError(t, err)
-	assert.Empty(t, leftover)
 }
 
 //nolint:paralleltest // changes the process-global working directory

internal/cli/freshen.go

@@ -33,9 +33,9 @@ var (
 	// modification time where one is required to carry it forward unchanged.
 	errEntryMissingMtime = errors.New("manifest entry has no mtime")
 
-	// errEntryMissingHash indicates a manifest entry that carries no hash
+	// errEntryMissingHash indicates a manifest entry that carries no
-	// to carry forward.
+	// SHA-256 hash to carry forward.
-	errEntryMissingHash = errors.New("manifest entry has no hash")
+	errEntryMissingHash = errors.New("manifest entry has no SHA-256 hash")
 )
 
 // errSignKeyNeeded indicates a signed manifest given to freshen with no
@@ -699,26 +699,26 @@ func addFileToBuilder(
 // Entries reach this path only when recordEntry classified them as
 // unchanged, which requires a recorded mtime, so an absent mtime here is
 // an error rather than something to paper over with the Unix epoch.
-// Loading refuses an entry with no hash; one that got here anyway is an
+// Only the entry's SHA-256 hash is carried over, wherever the entry lists
-// error too, not a file to leave out of the manifest.
+// it, since check and fetch compare with no other. Loading refuses an
+// entry with no SHA-256 hash; one that got here anyway is an error too,
+// not a file to leave out of the manifest.
 func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
 	mtime, ok := entryMtime(entry)
 	if !ok {
 		return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath())
 	}
 
-	if len(entry.GetHashes()) == 0 {
+	hash := mfer.SHA256Multihash(entry)
+	if hash == nil {
 		return fmt.Errorf("%w: %s", errEntryMissingHash, entry.GetPath())
 	}
 
 	err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
 		mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
-		fs.FileMode(entry.GetMode()), entry.GetHashes()[0].GetMultiHash())
+		fs.FileMode(entry.GetMode()), hash)
 	if err != nil {
-		return fmt.Errorf(
+		return fmt.Errorf("manifest entry %s: %w", entry.GetPath(), err)
-			"manifest entry %s: %w (regenerate the manifest with mfer generate)",
-			entry.GetPath(), err,
-		)
 	}
 
 	return nil

internal/cli/freshen_test.go

@@ -2,7 +2,10 @@
 package cli
 
 import (
+	"bytes"
+	"context"
 	"crypto/sha256"
+	"crypto/sha512"
 	"maps"
 	"os"
 	"path/filepath"
@@ -584,48 +587,76 @@ func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
 	assert.Contains(t, err.Error(), "file1.txt")
 }
 
-// TestFreshenAddExistingRejectsMissingHash pins that an entry with no hash
+// TestFreshenAddExistingRejectsMissingHash pins that an entry with no
-// is an error rather than a file left out of the rebuilt manifest.
+// SHA-256 hash, whether it lists no hash or only a SHA-1 one, is an error
+// rather than a file left out of the rebuilt manifest.
 func TestFreshenAddExistingRejectsMissingHash(t *testing.T) {
 	t.Parallel()
 
-	b := mfer.NewBuilder()
-	entry := &mfer.MFFilePath{
-		Path:  "data.bin",
-		Size:  8,
-		Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
-	}
-
-	err := addExistingToBuilder(b, entry)
-	require.ErrorIs(t, err, errEntryMissingHash)
-	assert.Contains(t, err.Error(), "data.bin")
-	assert.Zero(t, b.FileCount())
-}
-
-// TestFreshenAddExistingRejectsShortHash pins that an existing manifest
-// entry whose hash the builder refuses is reported as a problem with the
-// manifest, with the command that fixes it.
-func TestFreshenAddExistingRejectsShortHash(t *testing.T) {
-	t.Parallel()
-
 	sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
 	require.NoError(t, err)
 
+	tests := []struct {
+		name   string
+		hashes []*mfer.MFFileChecksum
+	}{
+		{"no hash", nil},
+		{"SHA-1 hash only", []*mfer.MFFileChecksum{{MultiHash: sha1Hash}}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			b := mfer.NewBuilder()
+			entry := &mfer.MFFilePath{
+				Path:   "data.bin",
+				Size:   8,
+				Mtime:  &mfer.Timestamp{Seconds: 1_700_000_000},
+				Hashes: tt.hashes,
+			}
+
+			err := addExistingToBuilder(b, entry)
+			require.ErrorIs(t, err, errEntryMissingHash)
+			assert.Contains(t, err.Error(), "data.bin")
+			assert.Zero(t, b.FileCount())
+		})
+	}
+}
+
+// TestFreshenAddExistingKeepsSHA256Hash pins that an unchanged entry that
+// lists another hash before its SHA-256 hash is carried over with the
+// SHA-256 hash, the one check and fetch compare with, and nothing else.
+func TestFreshenAddExistingKeepsSHA256Hash(t *testing.T) {
+	t.Parallel()
+
+	sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
+	require.NoError(t, err)
+
+	sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
+	require.NoError(t, err)
+
 	b := mfer.NewBuilder()
 	entry := &mfer.MFFilePath{
-		Path:  "old.txt",
+		Path:  "kept.txt",
-		Size:  8,
+		Size:  6,
 		Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
 		Hashes: []*mfer.MFFileChecksum{
-			{MultiHash: sha1Hash},
+			{MultiHash: sha512Hash},
+			{MultiHash: sha256Hash},
 		},
 	}
 
-	err = addExistingToBuilder(b, entry)
+	require.NoError(t, addExistingToBuilder(b, entry))
-	require.Error(t, err)
+
-	assert.Contains(t, err.Error(), "manifest entry old.txt")
+	var buf bytes.Buffer
-	assert.Contains(t, err.Error(), "mfer generate")
+	require.NoError(t, b.Build(context.Background(), &buf))
-	assert.Zero(t, b.FileCount())
+
+	m, err := mfer.NewManifestFromReader(&buf)
+	require.NoError(t, err)
+	require.Len(t, m.Files(), 1)
+	require.Len(t, m.Files()[0].GetHashes(), 1)
+	assert.Equal(t, sha256Hash, m.Files()[0].GetHashes()[0].GetMultiHash())
 }
 
 // TestEntryMtime pins the presence semantics the callers depend on.

mfer/builder.go

@@ -37,8 +37,7 @@ var (
 	errPathDot          = errors.New("contains '.' segment")
 	errSizeMismatch     = errors.New("size mismatch")
 	errNegativeSize     = errors.New("size cannot be negative")
-	errHashNotMultihash = errors.New("hash is not a valid multihash")
+	errHashNotSHA256    = errors.New("hash is not a SHA-256 multihash")
-	errHashTooShort     = errors.New("hash digest is too short")
 	errDuplicatePath    = errors.New("duplicate path")
 )
 
@@ -113,6 +112,28 @@ func (m ModTime) Timestamp() *Timestamp {
 // Multihash represents a multihash-encoded file hash (typically SHA2-256).
 type Multihash []byte
 
+// SHA256Multihash returns the first of entry's hashes that is a SHA-256
+// multihash, or nil if none is. Loading a manifest refuses an entry that
+// does not list exactly one, so for a loaded entry it is the only one.
+func SHA256Multihash(entry *MFFilePath) Multihash {
+	for _, h := range entry.GetHashes() {
+		if isSHA256Multihash(h.GetMultiHash()) {
+			return h.GetMultiHash()
+		}
+	}
+
+	return nil
+}
+
+// isSHA256Multihash reports whether hash is a SHA-256 multihash: the
+// multihash code for SHA-256, then a 32-byte digest.
+func isSHA256Multihash(hash []byte) bool {
+	decoded, err := multihash.Decode(hash)
+
+	return err == nil && decoded.Code == multihash.SHA2_256 &&
+		len(decoded.Digest) == sha256.Size
+}
+
 // FileHashProgress reports progress during file hashing.
 type FileHashProgress struct {
 	BytesRead FileSize // Total bytes read so far for the current file
@@ -241,8 +262,7 @@ func (b *Builder) FileCount() int {
 // This is useful when the hash is already known (e.g., from an existing manifest).
 // Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
 // Returns an error if path is invalid or already added, size is negative,
-// or hash is not a multihash with a digest of at least 32 bytes, as long
+// or hash is not a SHA-256 multihash, the hash every entry must carry.
-// as SHA-256's.
 func (b *Builder) AddFileWithHash(
 	path RelFilePath,
 	size FileSize,
@@ -259,19 +279,8 @@ func (b *Builder) AddFileWithHash(
 		return errNegativeSize
 	}
 
-	decoded, err := multihash.Decode(hash)
+	if !isSHA256Multihash(hash) {
-	if err != nil {
+		return errHashNotSHA256
-		return fmt.Errorf("%w: %w", errHashNotMultihash, err)
-	}
-
-	// The reader's limit on decoding cost (maxDecodedGrowth) assumes every
-	// hash is at least as long as a SHA-256 multihash, so a manifest of
-	// shorter ones could fail to load.
-	if len(decoded.Digest) < sha256.Size {
-		return fmt.Errorf(
-			"%w: %d bytes, at least %d needed",
-			errHashTooShort, len(decoded.Digest), sha256.Size,
-		)
 	}
 
 	entry := &MFFilePath{

mfer/builder_test.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"crypto/sha512"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -94,23 +95,25 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
 	sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
 	require.NoError(t, err)
 
+	sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
+	require.NoError(t, err)
+
 	truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
 	require.NoError(t, err)
 
 	tests := []struct {
 		name string
 		hash Multihash
-		want error
 	}{
-		{"nil hash", nil, errHashNotMultihash},
+		{"nil hash", nil},
-		{"empty hash", []byte{}, errHashNotMultihash},
+		{"empty hash", []byte{}},
-		{"one-byte hash", []byte{0x12}, errHashNotMultihash},
+		{"one-byte hash", []byte{0x12}},
 		// A SHA-256 code and 32-byte length, then only two bytes of digest.
-		{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
+		{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}},
-		// A valid multihash, but its 20-byte SHA-1 digest is too short.
+		{"SHA-1 multihash", sha1Hash},
-		{"SHA-1 multihash", sha1Hash, errHashTooShort},
+		{"SHA-512 multihash", sha512Hash},
-		// A valid multihash whose 31-byte digest is one byte short.
+		// A SHA-256 code with a 31-byte digest.
-		{"31-byte digest", truncatedHash, errHashTooShort},
+		{"31-byte digest", truncatedHash},
 	}
 
 	for _, tt := range tests {
@@ -119,7 +122,7 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
 
 			b := NewBuilder()
 			err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), 0, tt.hash)
-			require.ErrorIs(t, err, tt.want)
+			require.ErrorIs(t, err, errHashNotSHA256)
 			assert.Equal(t, 0, b.FileCount())
 		})
 	}

mfer/checker.go

@@ -80,7 +80,8 @@ type Checker struct {
 	// manifestPaths is a set of paths in the manifest for quick lookup
 	manifestPaths map[RelFilePath]struct{}
 	// manifestInfo is the manifest file, which FindExtraFiles leaves out,
-	// matched with os.SameFile.
+	// matched with os.SameFile. It is nil for a manifest read from
+	// CheckerOptions.ManifestReader.
 	manifestInfo os.FileInfo
 	// signature info from the manifest
 	signature     []byte
@@ -90,8 +91,13 @@ type Checker struct {
 
 // CheckerOptions configures a Checker.
 type CheckerOptions struct {
-	// ManifestPath is the manifest file to check against (required).
+	// ManifestPath is the manifest file to check against. It is required
+	// unless ManifestReader is given.
 	ManifestPath string
+	// ManifestReader, if not nil, is read for the manifest to check against
+	// in place of ManifestPath, such as a manifest downloaded into memory.
+	// FindExtraFiles then has no manifest file to leave out.
+	ManifestReader io.Reader
 	// BasePath is the directory relative to which manifest paths are
 	// resolved (required).
 	BasePath string
@@ -100,9 +106,10 @@ type CheckerOptions struct {
 }
 
 // NewChecker creates a new Checker with the given options. It returns an
-// error if opts is nil or either path is empty.
+// error if opts is nil, the base path is empty, or neither a manifest path
+// nor a manifest reader is given.
 func NewChecker(opts *CheckerOptions) (*Checker, error) {
-	if opts == nil || opts.ManifestPath == "" {
+	if opts == nil || (opts.ManifestPath == "" && opts.ManifestReader == nil) {
 		return nil, errManifestPathEmpty
 	}
 
@@ -115,12 +122,30 @@ func NewChecker(opts *CheckerOptions) (*Checker, error) {
 		fs = afero.NewOsFs()
 	}
 
-	m, err := NewManifestFromFile(&ManifestFromFileOptions{
+	var (
-		Path: opts.ManifestPath,
+		m            *manifest
-		Fs:   fs,
+		manifestInfo os.FileInfo
-	})
+		err          error
-	if err != nil {
+	)
-		return nil, err
+
+	if opts.ManifestReader != nil {
+		m, err = NewManifestFromReader(opts.ManifestReader)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		m, err = NewManifestFromFile(&ManifestFromFileOptions{
+			Path: opts.ManifestPath,
+			Fs:   fs,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		manifestInfo, err = fs.Stat(opts.ManifestPath)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	abs, err := filepath.Abs(opts.BasePath)
@@ -135,11 +160,6 @@ func NewChecker(opts *CheckerOptions) (*Checker, error) {
 		manifestPaths[RelFilePath(f.GetPath())] = struct{}{}
 	}
 
-	manifestInfo, err := fs.Stat(opts.ManifestPath)
-	if err != nil {
-		return nil, err
-	}
-
 	return &Checker{
 		basePath:      AbsFilePath(abs),
 		files:         files,

mfer/checker_test.go

@@ -189,6 +189,38 @@ func TestNewCheckerNilFsUsesOsFs(t *testing.T) {
 	assert.Equal(t, StatusOK, (<-results).Status)
 }
 
+// TestNewCheckerReadsManifestFromReader gives NewChecker the manifest as a
+// reader and no manifest path, on a filesystem that holds only the files it
+// lists: it checks those files and finds nothing unlisted.
+func TestNewCheckerReadsManifestFromReader(t *testing.T) {
+	t.Parallel()
+
+	files := map[string][]byte{testFile1: []byte("hello")}
+
+	manifestFs := afero.NewMemMapFs()
+	createTestManifest(t, manifestFs, testManifestPath, files)
+	manifestData, err := afero.ReadFile(manifestFs, testManifestPath)
+	require.NoError(t, err)
+
+	fs := afero.NewMemMapFs()
+	createFilesOnDisk(t, fs, files)
+
+	chk, err := NewChecker(&CheckerOptions{
+		ManifestReader: bytes.NewReader(manifestData),
+		BasePath:       testDataDir,
+		Fs:             fs,
+	})
+	require.NoError(t, err)
+
+	results := make(chan Result, 1)
+	require.NoError(t, chk.Check(context.Background(), results, nil))
+	assert.Equal(t, Result{Path: testFile1, Status: StatusOK}, <-results)
+
+	extras := make(chan Result, len(files))
+	require.NoError(t, chk.FindExtraFiles(context.Background(), extras))
+	assert.Empty(t, extras)
+}
+
 func TestCheckerFileCountAndTotalBytes(t *testing.T) {
 	t.Parallel()
 

mfer/deserialize.go

@@ -27,7 +27,8 @@ var (
 	errUUIDMismatch         = errors.New("outer and inner UUID mismatch")
 	errInvalidFileFormat    = errors.New("invalid file format")
 	errInvalidManifestPath  = errors.New("invalid file entry")
-	errEntryMissingHash     = errors.New("file entry has no hash")
+	errEntryMissingHash     = errors.New("file entry has no SHA-256 hash")
+	errEntryExtraSHA256Hash = errors.New("file entry has more than one SHA-256 hash")
 	errDecodedTooLarge      = errors.New("too much memory needed")
 	errSignerNotSigningKey  = errors.New(
 		"signer is not the fingerprint of the key that made the signature")
@@ -236,6 +237,30 @@ func forEachBytesField(
 	return nil
 }
 
+// validateEntryHashes refuses, naming its path, an entry that does not list
+// exactly one SHA-256 multihash; hashes of other algorithms beside it are
+// allowed. check and fetch compute only SHA-256, so with none they would
+// have nothing to check the file against, and with two check would pass a
+// file that matches either.
+func validateEntryHashes(entry *MFFilePath) error {
+	sha256Hashes := 0
+
+	for _, h := range entry.GetHashes() {
+		if isSHA256Multihash(h.GetMultiHash()) {
+			sha256Hashes++
+		}
+	}
+
+	switch {
+	case sha256Hashes == 0:
+		return fmt.Errorf("%w: %q", errEntryMissingHash, entry.GetPath())
+	case sha256Hashes > 1:
+		return fmt.Errorf("%w: %q", errEntryExtraSHA256Hash, entry.GetPath())
+	default:
+		return nil
+	}
+}
+
 func (m *manifest) deserializeInner() error {
 	err := m.validateOuterHeader()
 	if err != nil {
@@ -294,8 +319,8 @@ func (m *manifest) deserializeInner() error {
 	// untrusted .mf. Reject loudly on the first offender rather than
 	// dropping entries, which would let a hostile manifest hide files from a
 	// check. A path listed twice is refused too: check would check the one
-	// file against both entries. So is an entry with no hash: there is
+	// file against both entries. So is an entry that does not list exactly
-	// nothing to check the file against.
+	// one SHA-256 hash.
 	seen := make(map[string]bool, len(m.pbInner.GetFiles()))
 
 	for _, f := range m.pbInner.GetFiles() {
@@ -304,8 +329,9 @@ func (m *manifest) deserializeInner() error {
 			return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
 		}
 
-		if len(f.GetHashes()) == 0 {
+		err = validateEntryHashes(f)
-			return fmt.Errorf("%w: %q", errEntryMissingHash, f.GetPath())
+		if err != nil {
+			return err
 		}
 
 		if seen[f.GetPath()] {

mfer/deserialize_path_test.go

@@ -5,8 +5,10 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"crypto/sha512"
 	"fmt"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 	"uuid"
@@ -167,40 +169,94 @@ func TestDeserializeRefusesPathListedTwice(t *testing.T) {
 	}
 }
 
-// A manifest holding a file entry with no hash is refused as it is loaded,
+// A file entry must list exactly one SHA-256 hash, and may list hashes of
-// naming the path: there is nothing to check the file against.
+// other algorithms beside it. An entry with none, even one that lists a
-func TestDeserializeRefusesEntryWithNoHash(t *testing.T) {
+// SHA-512 hash, is refused as it is loaded, and so is an entry with two;
+// each refusal names the path. An entry listing a SHA-512 hash and then a
+// SHA-256 one loads with both.
+func TestDeserializeRequiresExactlyOneSHA256Hash(t *testing.T) {
 	t.Parallel()
 
-	id := uuid.NewV4()
+	sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
-	inner, err := proto.Marshal(&MFFile{
-		Version: MFFile_VERSION_ONE,
-		Files:   []*MFFilePath{{Path: "x", Size: 1}},
-		Uuid:    id[:],
-	})
 	require.NoError(t, err)
 
-	_, err = NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
+	otherSHA256Hash, err := multihash.Encode(
-	require.ErrorIs(t, err, errEntryMissingHash)
+		bytes.Repeat([]byte{1}, sha256.Size), multihash.SHA2_256)
-	require.EqualError(t, err, `file entry has no hash: "x"`)
+	require.NoError(t, err)
+
+	sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
+	require.NoError(t, err)
+
+	tests := []struct {
+		name   string
+		hashes [][]byte
+		want   error // nil for an entry that loads
+	}{
+		{"no hash", nil, errEntryMissingHash},
+		{"SHA-512 hash only", [][]byte{sha512Hash}, errEntryMissingHash},
+		{
+			"two SHA-256 hashes",
+			[][]byte{sha256Hash, otherSHA256Hash},
+			errEntryExtraSHA256Hash,
+		},
+		{"SHA-512 hash then SHA-256 hash", [][]byte{sha512Hash, sha256Hash}, nil},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			entry := &MFFilePath{Path: "x", Size: 1}
+			for _, h := range tt.hashes {
+				entry.Hashes = append(entry.Hashes, &MFFileChecksum{MultiHash: h})
+			}
+
+			id := uuid.NewV4()
+			inner, err := proto.Marshal(&MFFile{
+				Version: MFFile_VERSION_ONE,
+				Files:   []*MFFilePath{entry},
+				Uuid:    id[:],
+			})
+			require.NoError(t, err)
+
+			m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
+			if tt.want == nil {
+				require.NoError(t, err)
+				require.Len(t, m.Files(), 1)
+				assert.Len(t, m.Files()[0].GetHashes(), len(tt.hashes))
+
+				return
+			}
+
+			require.ErrorIs(t, err, tt.want)
+			assert.Contains(t, err.Error(), `"x"`)
+		})
+	}
 }
 
-// Entries of a path, an empty hash, an empty MIME type and empty modification
+// Entries of a path, a SHA-256 hash, an empty MIME type and empty
-// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
+// modification and change times are counted at 432 bytes each (176 + 112 +
-// and take 16 bytes plus the path to encode. A 37-character path makes that
+// 16 + 64 + 64) and take 52 bytes plus the path to encode. A one-character
-// 53 bytes, about 8.2 times: refused, and leaving any one of the five
+// path makes that 53 bytes, about 8.2 times: refused, and leaving any one of
-// uncounted, even the MIME type, brings it under 8. A 39-character path makes
+// the five uncounted, even the MIME type, brings it under 8. A two-character
-// it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
+// path makes it 54 bytes, 8 times exactly: loaded. There is one entry for
-// padded with zeros to that length, since a manifest lists a path only once.
+// each base-36 digit, and each entry's path is its number in base 36, padded
+// with zeros to that length, since a manifest lists a path only once.
 func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
 	t.Parallel()
 
+	hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
+	require.NoError(t, err)
+
+	checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
+	require.NoError(t, err)
+
 	tests := []struct {
 		pathLen int
 		refused bool
 	}{
-		{37, true},
+		{1, true},
-		{39, false},
+		{2, false},
 	}
 
 	for _, tt := range tests {
@@ -213,11 +269,14 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
 			inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
 			inner = protowire.AppendBytes(inner, id[:])
 
-			for i := range 1000 {
+			for i := range 36 {
+				path := strconv.FormatInt(int64(i), 36)
+				path = strings.Repeat("0", tt.pathLen-len(path)) + path
+
 				entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
-				entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
+				entry = protowire.AppendString(entry, path)
 				entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
-				entry = protowire.AppendBytes(entry, nil)
+				entry = protowire.AppendBytes(entry, checksum)
 				entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
 				entry = protowire.AppendBytes(entry, nil)
 				entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
@@ -247,10 +306,16 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
 	unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
 	unknown = protowire.AppendBytes(unknown, []byte("not known"))
 
+	hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
+	require.NoError(t, err)
+
+	checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
+	require.NoError(t, err)
+
 	entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
 	entry = protowire.AppendString(entry, "a")
 	entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
-	entry = protowire.AppendBytes(entry, nil)
+	entry = protowire.AppendBytes(entry, checksum)
 	entry = append(entry, unknown...)
 
 	id := uuid.NewV4()