Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
59b3cbd1dd |
No files matched your search
+8
-17
@@ -81,10 +81,10 @@ After decompressing `innerMessage`, the result is a serialized `MFFile`
|
|||||||
Each file entry contains:
|
Each file entry contains:
|
||||||
|
|
||||||
| Field | Number | Type | Description |
|
| Field | Number | Type | Description |
|
||||||
| ---------- | ------ | ------------------------- | ------------------------------------------------------------ |
|
| ---------- | ------ | ------------------------- | ------------------------------------------------------------- |
|
||||||
| `path` | 1 | string | Relative file path (see Path Rules) |
|
| `path` | 1 | string | Relative file path (see Path Rules) |
|
||||||
| `size` | 2 | int64 | File size in bytes |
|
| `size` | 2 | int64 | File size in bytes |
|
||||||
| `hashes` | 3 | repeated `MFFileChecksum` | Must include exactly one SHA-256 multihash (see Hash Format) |
|
| `hashes` | 3 | repeated `MFFileChecksum` | At least one hash required; readers reject an entry with none |
|
||||||
| `mimeType` | 301 | string (optional) | MIME type |
|
| `mimeType` | 301 | string (optional) | MIME type |
|
||||||
| `mtime` | 302 | Timestamp (optional) | Modification time |
|
| `mtime` | 302 | Timestamp (optional) | Modification time |
|
||||||
| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) |
|
| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) |
|
||||||
@@ -133,16 +133,8 @@ Each checksum is a single `bytes multiHash` field containing a
|
|||||||
self-describing: the encoded bytes include a varint algorithm identifier
|
self-describing: the encoded bytes include a varint algorithm identifier
|
||||||
followed by a varint digest length followed by the digest itself.
|
followed by a varint digest length followed by the digest itself.
|
||||||
|
|
||||||
A writer must give every file entry exactly one SHA-256 multihash of the file's
|
The 1.0 implementation writes SHA-256 multihashes (`0x12` algorithm code).
|
||||||
content: the algorithm code `0x12`, the digest length `0x20` and the 32 bytes of
|
Implementations must be able to verify SHA-256 multihashes at minimum.
|
||||||
the digest. It may list multihashes of other algorithms beside it. A reader must
|
|
||||||
refuse a manifest with an entry that has no SHA-256 multihash or more than one,
|
|
||||||
whatever other hashes the entry lists. A verifier then needs only SHA-256,
|
|
||||||
checking each file against its entry's SHA-256 multihash; it may ignore the
|
|
||||||
other hashes.
|
|
||||||
|
|
||||||
The reference implementation writes one hash for each entry, the SHA-256
|
|
||||||
multihash, and its `check` and `fetch` compute only SHA-256.
|
|
||||||
|
|
||||||
## Signature Scheme
|
## Signature Scheme
|
||||||
|
|
||||||
@@ -220,11 +212,10 @@ compresses the same inner message to the same bytes:
|
|||||||
- Each entry holds only `path`, `size`, `hashes` with exactly one hash, `mtime`
|
- Each entry holds only `path`, `size`, `hashes` with exactly one hash, `mtime`
|
||||||
and `mode`. The hash is the SHA-256 multihash of the file's content, except
|
and `mode`. The hash is the SHA-256 multihash of the file's content, except
|
||||||
that `freshen` reads only the files that are new or whose size, `mtime` or
|
that `freshen` reads only the files that are new or whose size, `mtime` or
|
||||||
`mode` changed, and for every other file carries the SHA-256 multihash of its
|
`mode` changed, and for every other file carries the first hash of its
|
||||||
existing entry over unchanged, leaving out any other hash the entry lists.
|
existing entry over unchanged. `mtime` is the file's modification time as the
|
||||||
`mtime` is the file's modification time as the file system gives it, to the
|
file system gives it, to the nanosecond, and `mode` is `0000` unless `gen` or
|
||||||
nanosecond, and `mode` is `0000` unless `gen` or `freshen` is given
|
`freshen` is given `--include-permissions`.
|
||||||
`--include-permissions`.
|
|
||||||
- `createdAt` is left out unless `gen` or `freshen` is given
|
- `createdAt` is left out unless `gen` or `freshen` is given
|
||||||
`--include-timestamps`. Then it holds the time the run started building the
|
`--include-timestamps`. Then it holds the time the run started building the
|
||||||
manifest, to the nanosecond, so it differs on every run.
|
manifest, to the nanosecond, so it differs on every run.
|
||||||
|
|||||||
@@ -121,10 +121,8 @@ func (mfa *CLIApp) downloadManifest(ctx context.Context, url string) ([]byte, er
|
|||||||
}
|
}
|
||||||
|
|
||||||
// verifyRequiredSigner enforces the --require-signature fingerprint
|
// verifyRequiredSigner enforces the --require-signature fingerprint
|
||||||
// against signer, the fingerprint of the key that made the manifest's
|
// against the key that made the manifest's signature.
|
||||||
// signature, which is empty when the manifest is not signed. Loading the
|
func verifyRequiredSigner(chk *mfer.Checker, requiredSigner string) error {
|
||||||
// manifest checked that signer is that key's fingerprint.
|
|
||||||
func verifyRequiredSigner(signer, requiredSigner string) error {
|
|
||||||
// Validate fingerprint format: must be exactly 40 hex characters
|
// Validate fingerprint format: must be exactly 40 hex characters
|
||||||
if len(requiredSigner) != fingerprintHexLen {
|
if len(requiredSigner) != fingerprintHexLen {
|
||||||
return fmt.Errorf("%w, got %d", errInvalidFingerprint, len(requiredSigner))
|
return fmt.Errorf("%w, got %d", errInvalidFingerprint, len(requiredSigner))
|
||||||
@@ -135,11 +133,15 @@ func verifyRequiredSigner(signer, requiredSigner string) error {
|
|||||||
return fmt.Errorf("invalid fingerprint: %w", err)
|
return fmt.Errorf("invalid fingerprint: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if signer == "" {
|
if !chk.IsSigned() {
|
||||||
return fmt.Errorf("%w, but signature from %s is required",
|
return fmt.Errorf("%w, but signature from %s is required",
|
||||||
errManifestNotSigned, requiredSigner)
|
errManifestNotSigned, requiredSigner)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Loading the manifest checked that the signer is the fingerprint of
|
||||||
|
// the key that made the signature.
|
||||||
|
signer := string(chk.Signer())
|
||||||
|
|
||||||
// Compare fingerprints - must be exact match (case-insensitive)
|
// Compare fingerprints - must be exact match (case-insensitive)
|
||||||
if !strings.EqualFold(signer, requiredSigner) {
|
if !strings.EqualFold(signer, requiredSigner) {
|
||||||
return fmt.Errorf("embedded signing key fingerprint %s %w %s",
|
return fmt.Errorf("embedded signing key fingerprint %s %w %s",
|
||||||
@@ -314,7 +316,7 @@ func (mfa *CLIApp) checkManifestOperation(
|
|||||||
// Check signature requirement. An empty fingerprint is checked too,
|
// Check signature requirement. An empty fingerprint is checked too,
|
||||||
// and refused as one of the wrong length.
|
// and refused as one of the wrong length.
|
||||||
if cmd.IsSet(flagRequireSignature) {
|
if cmd.IsSet(flagRequireSignature) {
|
||||||
err = verifyRequiredSigner(string(chk.Signer()), cmd.String(flagRequireSignature))
|
err = verifyRequiredSigner(chk, cmd.String(flagRequireSignature))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,6 +50,38 @@ func runLocked(fn func() error) error {
|
|||||||
return fn()
|
return fn()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unsignedChecker builds a Checker over a freshly scanned, unsigned manifest.
|
||||||
|
func unsignedChecker(t *testing.T) *mfer.Checker {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
// The scan and the manifest load log to the process-global logger, so
|
||||||
|
// serialize with the CLI runs.
|
||||||
|
runMu.Lock()
|
||||||
|
defer runMu.Unlock()
|
||||||
|
|
||||||
|
fs := afero.NewMemMapFs()
|
||||||
|
require.NoError(t, fs.MkdirAll("/d", 0o755))
|
||||||
|
require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
|
||||||
|
|
||||||
|
s := mfer.NewScannerWithOptions(&mfer.ScannerOptions{Fs: fs})
|
||||||
|
require.NoError(t, s.EnumeratePath("/d", nil))
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
|
||||||
|
require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644))
|
||||||
|
|
||||||
|
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
||||||
|
ManifestPath: "/d/index.mf",
|
||||||
|
BasePath: "/d",
|
||||||
|
Fs: fs,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.False(t, chk.IsSigned())
|
||||||
|
|
||||||
|
return chk
|
||||||
|
}
|
||||||
|
|
||||||
func TestNoManifestFoundMessage(t *testing.T) {
|
func TestNoManifestFoundMessage(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -65,7 +97,7 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
|
|||||||
t.Run("invalid fingerprint length", func(t *testing.T) {
|
t.Run("invalid fingerprint length", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
err := verifyRequiredSigner("", "12345678")
|
err := verifyRequiredSigner(unsignedChecker(t), "12345678")
|
||||||
require.ErrorIs(t, err, errInvalidFingerprint)
|
require.ErrorIs(t, err, errInvalidFingerprint)
|
||||||
assert.EqualError(t, err,
|
assert.EqualError(t, err,
|
||||||
"invalid fingerprint: must be exactly 40 hex characters, got 8")
|
"invalid fingerprint: must be exactly 40 hex characters, got 8")
|
||||||
@@ -74,7 +106,7 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
|
|||||||
t.Run("manifest not signed", func(t *testing.T) {
|
t.Run("manifest not signed", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
err := verifyRequiredSigner("", msgFpA)
|
err := verifyRequiredSigner(unsignedChecker(t), msgFpA)
|
||||||
require.ErrorIs(t, err, errManifestNotSigned)
|
require.ErrorIs(t, err, errManifestNotSigned)
|
||||||
assert.EqualError(t, err,
|
assert.EqualError(t, err,
|
||||||
"manifest is not signed, but signature from "+msgFpA+" is required")
|
"manifest is not signed, but signature from "+msgFpA+" is required")
|
||||||
@@ -89,13 +121,13 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
|
|||||||
func TestSignerMismatchMessage(t *testing.T) {
|
func TestSignerMismatchMessage(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
signer := string(signedChecker(t,
|
chk := signedChecker(t,
|
||||||
signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")})).Signer())
|
signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")}))
|
||||||
|
|
||||||
err := verifyRequiredSigner(signer, msgFpB)
|
err := verifyRequiredSigner(chk, msgFpB)
|
||||||
require.ErrorIs(t, err, errSignerMismatch)
|
require.ErrorIs(t, err, errSignerMismatch)
|
||||||
assert.EqualError(t, err,
|
assert.EqualError(t, err,
|
||||||
"embedded signing key fingerprint "+signer+
|
"embedded signing key fingerprint "+string(chk.Signer())+
|
||||||
" does not match required "+msgFpB)
|
" does not match required "+msgFpB)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+16
-1
@@ -474,7 +474,7 @@ func (mfa *CLIApp) fetchManifest(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if requiredSigner != "" {
|
if requiredSigner != "" {
|
||||||
err = verifyRequiredSigner(string(manifest.Signer()), requiredSigner)
|
err = verifyFetchedSigner(manifestData, requiredSigner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
@@ -552,6 +552,21 @@ func checkNoNameClash(files []*mfer.MFFilePath) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// verifyFetchedSigner enforces --require-signature on the fetched manifest
|
||||||
|
// exactly as check does. The Checker it builds is used only for the
|
||||||
|
// manifest's signature, so its base path is never read.
|
||||||
|
func verifyFetchedSigner(manifestData []byte, requiredSigner string) error {
|
||||||
|
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
||||||
|
ManifestReader: bytes.NewReader(manifestData),
|
||||||
|
BasePath: "/",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("load manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return verifyRequiredSigner(chk, requiredSigner)
|
||||||
|
}
|
||||||
|
|
||||||
// saveManifest writes the fetched manifest into dest under the default
|
// saveManifest writes the fetched manifest into dest under the default
|
||||||
// manifest name, the way fetch writes every file: to a new temp file that
|
// manifest name, the way fetch writes every file: to a new temp file that
|
||||||
// is then renamed into place.
|
// is then renamed into place.
|
||||||
|
|||||||
+11
-11
@@ -33,9 +33,9 @@ var (
|
|||||||
// modification time where one is required to carry it forward unchanged.
|
// modification time where one is required to carry it forward unchanged.
|
||||||
errEntryMissingMtime = errors.New("manifest entry has no mtime")
|
errEntryMissingMtime = errors.New("manifest entry has no mtime")
|
||||||
|
|
||||||
// errEntryMissingHash indicates a manifest entry that carries no
|
// errEntryMissingHash indicates a manifest entry that carries no hash
|
||||||
// SHA-256 hash to carry forward.
|
// to carry forward.
|
||||||
errEntryMissingHash = errors.New("manifest entry has no SHA-256 hash")
|
errEntryMissingHash = errors.New("manifest entry has no hash")
|
||||||
)
|
)
|
||||||
|
|
||||||
// errSignKeyNeeded indicates a signed manifest given to freshen with no
|
// errSignKeyNeeded indicates a signed manifest given to freshen with no
|
||||||
@@ -699,26 +699,26 @@ func addFileToBuilder(
|
|||||||
// Entries reach this path only when recordEntry classified them as
|
// Entries reach this path only when recordEntry classified them as
|
||||||
// unchanged, which requires a recorded mtime, so an absent mtime here is
|
// unchanged, which requires a recorded mtime, so an absent mtime here is
|
||||||
// an error rather than something to paper over with the Unix epoch.
|
// an error rather than something to paper over with the Unix epoch.
|
||||||
// Only the entry's SHA-256 hash is carried over, wherever the entry lists
|
// Loading refuses an entry with no hash; one that got here anyway is an
|
||||||
// it, since check and fetch compare with no other. Loading refuses an
|
// error too, not a file to leave out of the manifest.
|
||||||
// entry with no SHA-256 hash; one that got here anyway is an error too,
|
|
||||||
// not a file to leave out of the manifest.
|
|
||||||
func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
||||||
mtime, ok := entryMtime(entry)
|
mtime, ok := entryMtime(entry)
|
||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath())
|
return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath())
|
||||||
}
|
}
|
||||||
|
|
||||||
hash := mfer.SHA256Multihash(entry)
|
if len(entry.GetHashes()) == 0 {
|
||||||
if hash == nil {
|
|
||||||
return fmt.Errorf("%w: %s", errEntryMissingHash, entry.GetPath())
|
return fmt.Errorf("%w: %s", errEntryMissingHash, entry.GetPath())
|
||||||
}
|
}
|
||||||
|
|
||||||
err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
||||||
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
||||||
fs.FileMode(entry.GetMode()), hash)
|
fs.FileMode(entry.GetMode()), entry.GetHashes()[0].GetMultiHash())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("manifest entry %s: %w", entry.GetPath(), err)
|
return fmt.Errorf(
|
||||||
|
"manifest entry %s: %w (regenerate the manifest with mfer generate)",
|
||||||
|
entry.GetPath(), err,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -2,10 +2,7 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"crypto/sha512"
|
|
||||||
"maps"
|
"maps"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -587,76 +584,48 @@ func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
|
|||||||
assert.Contains(t, err.Error(), "file1.txt")
|
assert.Contains(t, err.Error(), "file1.txt")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFreshenAddExistingRejectsMissingHash pins that an entry with no
|
// TestFreshenAddExistingRejectsMissingHash pins that an entry with no hash
|
||||||
// SHA-256 hash, whether it lists no hash or only a SHA-1 one, is an error
|
// is an error rather than a file left out of the rebuilt manifest.
|
||||||
// rather than a file left out of the rebuilt manifest.
|
|
||||||
func TestFreshenAddExistingRejectsMissingHash(t *testing.T) {
|
func TestFreshenAddExistingRejectsMissingHash(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
hashes []*mfer.MFFileChecksum
|
|
||||||
}{
|
|
||||||
{"no hash", nil},
|
|
||||||
{"SHA-1 hash only", []*mfer.MFFileChecksum{{MultiHash: sha1Hash}}},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := mfer.NewBuilder()
|
b := mfer.NewBuilder()
|
||||||
entry := &mfer.MFFilePath{
|
entry := &mfer.MFFilePath{
|
||||||
Path: "data.bin",
|
Path: "data.bin",
|
||||||
Size: 8,
|
Size: 8,
|
||||||
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
||||||
Hashes: tt.hashes,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err := addExistingToBuilder(b, entry)
|
err := addExistingToBuilder(b, entry)
|
||||||
require.ErrorIs(t, err, errEntryMissingHash)
|
require.ErrorIs(t, err, errEntryMissingHash)
|
||||||
assert.Contains(t, err.Error(), "data.bin")
|
assert.Contains(t, err.Error(), "data.bin")
|
||||||
assert.Zero(t, b.FileCount())
|
assert.Zero(t, b.FileCount())
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFreshenAddExistingKeepsSHA256Hash pins that an unchanged entry that
|
// TestFreshenAddExistingRejectsShortHash pins that an existing manifest
|
||||||
// lists another hash before its SHA-256 hash is carried over with the
|
// entry whose hash the builder refuses is reported as a problem with the
|
||||||
// SHA-256 hash, the one check and fetch compare with, and nothing else.
|
// manifest, with the command that fixes it.
|
||||||
func TestFreshenAddExistingKeepsSHA256Hash(t *testing.T) {
|
func TestFreshenAddExistingRejectsShortHash(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
|
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
b := mfer.NewBuilder()
|
b := mfer.NewBuilder()
|
||||||
entry := &mfer.MFFilePath{
|
entry := &mfer.MFFilePath{
|
||||||
Path: "kept.txt",
|
Path: "old.txt",
|
||||||
Size: 6,
|
Size: 8,
|
||||||
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
||||||
Hashes: []*mfer.MFFileChecksum{
|
Hashes: []*mfer.MFFileChecksum{
|
||||||
{MultiHash: sha512Hash},
|
{MultiHash: sha1Hash},
|
||||||
{MultiHash: sha256Hash},
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, addExistingToBuilder(b, entry))
|
err = addExistingToBuilder(b, entry)
|
||||||
|
require.Error(t, err)
|
||||||
var buf bytes.Buffer
|
assert.Contains(t, err.Error(), "manifest entry old.txt")
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
assert.Contains(t, err.Error(), "mfer generate")
|
||||||
|
assert.Zero(t, b.FileCount())
|
||||||
m, err := mfer.NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, m.Files(), 1)
|
|
||||||
require.Len(t, m.Files()[0].GetHashes(), 1)
|
|
||||||
assert.Equal(t, sha256Hash, m.Files()[0].GetHashes()[0].GetMultiHash())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEntryMtime pins the presence semantics the callers depend on.
|
// TestEntryMtime pins the presence semantics the callers depend on.
|
||||||
|
|||||||
+17
-26
@@ -37,7 +37,8 @@ var (
|
|||||||
errPathDot = errors.New("contains '.' segment")
|
errPathDot = errors.New("contains '.' segment")
|
||||||
errSizeMismatch = errors.New("size mismatch")
|
errSizeMismatch = errors.New("size mismatch")
|
||||||
errNegativeSize = errors.New("size cannot be negative")
|
errNegativeSize = errors.New("size cannot be negative")
|
||||||
errHashNotSHA256 = errors.New("hash is not a SHA-256 multihash")
|
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
||||||
|
errHashTooShort = errors.New("hash digest is too short")
|
||||||
errDuplicatePath = errors.New("duplicate path")
|
errDuplicatePath = errors.New("duplicate path")
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -112,28 +113,6 @@ func (m ModTime) Timestamp() *Timestamp {
|
|||||||
// Multihash represents a multihash-encoded file hash (typically SHA2-256).
|
// Multihash represents a multihash-encoded file hash (typically SHA2-256).
|
||||||
type Multihash []byte
|
type Multihash []byte
|
||||||
|
|
||||||
// SHA256Multihash returns the first of entry's hashes that is a SHA-256
|
|
||||||
// multihash, or nil if none is. Loading a manifest refuses an entry that
|
|
||||||
// does not list exactly one, so for a loaded entry it is the only one.
|
|
||||||
func SHA256Multihash(entry *MFFilePath) Multihash {
|
|
||||||
for _, h := range entry.GetHashes() {
|
|
||||||
if isSHA256Multihash(h.GetMultiHash()) {
|
|
||||||
return h.GetMultiHash()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// isSHA256Multihash reports whether hash is a SHA-256 multihash: the
|
|
||||||
// multihash code for SHA-256, then a 32-byte digest.
|
|
||||||
func isSHA256Multihash(hash []byte) bool {
|
|
||||||
decoded, err := multihash.Decode(hash)
|
|
||||||
|
|
||||||
return err == nil && decoded.Code == multihash.SHA2_256 &&
|
|
||||||
len(decoded.Digest) == sha256.Size
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileHashProgress reports progress during file hashing.
|
// FileHashProgress reports progress during file hashing.
|
||||||
type FileHashProgress struct {
|
type FileHashProgress struct {
|
||||||
BytesRead FileSize // Total bytes read so far for the current file
|
BytesRead FileSize // Total bytes read so far for the current file
|
||||||
@@ -262,7 +241,8 @@ func (b *Builder) FileCount() int {
|
|||||||
// This is useful when the hash is already known (e.g., from an existing manifest).
|
// This is useful when the hash is already known (e.g., from an existing manifest).
|
||||||
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
||||||
// Returns an error if path is invalid or already added, size is negative,
|
// Returns an error if path is invalid or already added, size is negative,
|
||||||
// or hash is not a SHA-256 multihash, the hash every entry must carry.
|
// or hash is not a multihash with a digest of at least 32 bytes, as long
|
||||||
|
// as SHA-256's.
|
||||||
func (b *Builder) AddFileWithHash(
|
func (b *Builder) AddFileWithHash(
|
||||||
path RelFilePath,
|
path RelFilePath,
|
||||||
size FileSize,
|
size FileSize,
|
||||||
@@ -279,8 +259,19 @@ func (b *Builder) AddFileWithHash(
|
|||||||
return errNegativeSize
|
return errNegativeSize
|
||||||
}
|
}
|
||||||
|
|
||||||
if !isSHA256Multihash(hash) {
|
decoded, err := multihash.Decode(hash)
|
||||||
return errHashNotSHA256
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w: %w", errHashNotMultihash, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reader's limit on decoding cost (maxDecodedGrowth) assumes every
|
||||||
|
// hash is at least as long as a SHA-256 multihash, so a manifest of
|
||||||
|
// shorter ones could fail to load.
|
||||||
|
if len(decoded.Digest) < sha256.Size {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w: %d bytes, at least %d needed",
|
||||||
|
errHashTooShort, len(decoded.Digest), sha256.Size,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
entry := &MFFilePath{
|
entry := &MFFilePath{
|
||||||
|
|||||||
+10
-13
@@ -5,7 +5,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"crypto/sha512"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -95,25 +94,23 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
|||||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
|
truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
hash Multihash
|
hash Multihash
|
||||||
|
want error
|
||||||
}{
|
}{
|
||||||
{"nil hash", nil},
|
{"nil hash", nil, errHashNotMultihash},
|
||||||
{"empty hash", []byte{}},
|
{"empty hash", []byte{}, errHashNotMultihash},
|
||||||
{"one-byte hash", []byte{0x12}},
|
{"one-byte hash", []byte{0x12}, errHashNotMultihash},
|
||||||
// A SHA-256 code and 32-byte length, then only two bytes of digest.
|
// A SHA-256 code and 32-byte length, then only two bytes of digest.
|
||||||
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}},
|
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
|
||||||
{"SHA-1 multihash", sha1Hash},
|
// A valid multihash, but its 20-byte SHA-1 digest is too short.
|
||||||
{"SHA-512 multihash", sha512Hash},
|
{"SHA-1 multihash", sha1Hash, errHashTooShort},
|
||||||
// A SHA-256 code with a 31-byte digest.
|
// A valid multihash whose 31-byte digest is one byte short.
|
||||||
{"31-byte digest", truncatedHash},
|
{"31-byte digest", truncatedHash, errHashTooShort},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -122,7 +119,7 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
|||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), 0, tt.hash)
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), 0, tt.hash)
|
||||||
require.ErrorIs(t, err, errHashNotSHA256)
|
require.ErrorIs(t, err, tt.want)
|
||||||
assert.Equal(t, 0, b.FileCount())
|
assert.Equal(t, 0, b.FileCount())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
+5
-31
@@ -27,8 +27,7 @@ var (
|
|||||||
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
||||||
errInvalidFileFormat = errors.New("invalid file format")
|
errInvalidFileFormat = errors.New("invalid file format")
|
||||||
errInvalidManifestPath = errors.New("invalid file entry")
|
errInvalidManifestPath = errors.New("invalid file entry")
|
||||||
errEntryMissingHash = errors.New("file entry has no SHA-256 hash")
|
errEntryMissingHash = errors.New("file entry has no hash")
|
||||||
errEntryExtraSHA256Hash = errors.New("file entry has more than one SHA-256 hash")
|
|
||||||
errDecodedTooLarge = errors.New("too much memory needed")
|
errDecodedTooLarge = errors.New("too much memory needed")
|
||||||
errSignerNotSigningKey = errors.New(
|
errSignerNotSigningKey = errors.New(
|
||||||
"signer is not the fingerprint of the key that made the signature")
|
"signer is not the fingerprint of the key that made the signature")
|
||||||
@@ -237,30 +236,6 @@ func forEachBytesField(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateEntryHashes refuses, naming its path, an entry that does not list
|
|
||||||
// exactly one SHA-256 multihash; hashes of other algorithms beside it are
|
|
||||||
// allowed. check and fetch compute only SHA-256, so with none they would
|
|
||||||
// have nothing to check the file against, and with two check would pass a
|
|
||||||
// file that matches either.
|
|
||||||
func validateEntryHashes(entry *MFFilePath) error {
|
|
||||||
sha256Hashes := 0
|
|
||||||
|
|
||||||
for _, h := range entry.GetHashes() {
|
|
||||||
if isSHA256Multihash(h.GetMultiHash()) {
|
|
||||||
sha256Hashes++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case sha256Hashes == 0:
|
|
||||||
return fmt.Errorf("%w: %q", errEntryMissingHash, entry.GetPath())
|
|
||||||
case sha256Hashes > 1:
|
|
||||||
return fmt.Errorf("%w: %q", errEntryExtraSHA256Hash, entry.GetPath())
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *manifest) deserializeInner() error {
|
func (m *manifest) deserializeInner() error {
|
||||||
err := m.validateOuterHeader()
|
err := m.validateOuterHeader()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -319,8 +294,8 @@ func (m *manifest) deserializeInner() error {
|
|||||||
// untrusted .mf. Reject loudly on the first offender rather than
|
// untrusted .mf. Reject loudly on the first offender rather than
|
||||||
// dropping entries, which would let a hostile manifest hide files from a
|
// dropping entries, which would let a hostile manifest hide files from a
|
||||||
// check. A path listed twice is refused too: check would check the one
|
// check. A path listed twice is refused too: check would check the one
|
||||||
// file against both entries. So is an entry that does not list exactly
|
// file against both entries. So is an entry with no hash: there is
|
||||||
// one SHA-256 hash.
|
// nothing to check the file against.
|
||||||
seen := make(map[string]bool, len(m.pbInner.GetFiles()))
|
seen := make(map[string]bool, len(m.pbInner.GetFiles()))
|
||||||
|
|
||||||
for _, f := range m.pbInner.GetFiles() {
|
for _, f := range m.pbInner.GetFiles() {
|
||||||
@@ -329,9 +304,8 @@ func (m *manifest) deserializeInner() error {
|
|||||||
return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
|
return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = validateEntryHashes(f)
|
if len(f.GetHashes()) == 0 {
|
||||||
if err != nil {
|
return fmt.Errorf("%w: %q", errEntryMissingHash, f.GetPath())
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if seen[f.GetPath()] {
|
if seen[f.GetPath()] {
|
||||||
|
|||||||
@@ -5,10 +5,8 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"crypto/sha512"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
"uuid"
|
"uuid"
|
||||||
@@ -169,94 +167,40 @@ func TestDeserializeRefusesPathListedTwice(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A file entry must list exactly one SHA-256 hash, and may list hashes of
|
// A manifest holding a file entry with no hash is refused as it is loaded,
|
||||||
// other algorithms beside it. An entry with none, even one that lists a
|
// naming the path: there is nothing to check the file against.
|
||||||
// SHA-512 hash, is refused as it is loaded, and so is an entry with two;
|
func TestDeserializeRefusesEntryWithNoHash(t *testing.T) {
|
||||||
// each refusal names the path. An entry listing a SHA-512 hash and then a
|
|
||||||
// SHA-256 one loads with both.
|
|
||||||
func TestDeserializeRequiresExactlyOneSHA256Hash(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
otherSHA256Hash, err := multihash.Encode(
|
|
||||||
bytes.Repeat([]byte{1}, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
hashes [][]byte
|
|
||||||
want error // nil for an entry that loads
|
|
||||||
}{
|
|
||||||
{"no hash", nil, errEntryMissingHash},
|
|
||||||
{"SHA-512 hash only", [][]byte{sha512Hash}, errEntryMissingHash},
|
|
||||||
{
|
|
||||||
"two SHA-256 hashes",
|
|
||||||
[][]byte{sha256Hash, otherSHA256Hash},
|
|
||||||
errEntryExtraSHA256Hash,
|
|
||||||
},
|
|
||||||
{"SHA-512 hash then SHA-256 hash", [][]byte{sha512Hash, sha256Hash}, nil},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
entry := &MFFilePath{Path: "x", Size: 1}
|
|
||||||
for _, h := range tt.hashes {
|
|
||||||
entry.Hashes = append(entry.Hashes, &MFFileChecksum{MultiHash: h})
|
|
||||||
}
|
|
||||||
|
|
||||||
id := uuid.NewV4()
|
id := uuid.NewV4()
|
||||||
inner, err := proto.Marshal(&MFFile{
|
inner, err := proto.Marshal(&MFFile{
|
||||||
Version: MFFile_VERSION_ONE,
|
Version: MFFile_VERSION_ONE,
|
||||||
Files: []*MFFilePath{entry},
|
Files: []*MFFilePath{{Path: "x", Size: 1}},
|
||||||
Uuid: id[:],
|
Uuid: id[:],
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
_, err = NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
||||||
if tt.want == nil {
|
require.ErrorIs(t, err, errEntryMissingHash)
|
||||||
require.NoError(t, err)
|
require.EqualError(t, err, `file entry has no hash: "x"`)
|
||||||
require.Len(t, m.Files(), 1)
|
|
||||||
assert.Len(t, m.Files()[0].GetHashes(), len(tt.hashes))
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
require.ErrorIs(t, err, tt.want)
|
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
||||||
assert.Contains(t, err.Error(), `"x"`)
|
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
|
||||||
})
|
// and take 16 bytes plus the path to encode. A 37-character path makes that
|
||||||
}
|
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
|
||||||
}
|
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
|
||||||
|
// it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
|
||||||
// Entries of a path, a SHA-256 hash, an empty MIME type and empty
|
// padded with zeros to that length, since a manifest lists a path only once.
|
||||||
// modification and change times are counted at 432 bytes each (176 + 112 +
|
|
||||||
// 16 + 64 + 64) and take 52 bytes plus the path to encode. A one-character
|
|
||||||
// path makes that 53 bytes, about 8.2 times: refused, and leaving any one of
|
|
||||||
// the five uncounted, even the MIME type, brings it under 8. A two-character
|
|
||||||
// path makes it 54 bytes, 8 times exactly: loaded. There is one entry for
|
|
||||||
// each base-36 digit, and each entry's path is its number in base 36, padded
|
|
||||||
// with zeros to that length, since a manifest lists a path only once.
|
|
||||||
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
pathLen int
|
pathLen int
|
||||||
refused bool
|
refused bool
|
||||||
}{
|
}{
|
||||||
{1, true},
|
{37, true},
|
||||||
{2, false},
|
{39, false},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -269,14 +213,11 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|||||||
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
||||||
inner = protowire.AppendBytes(inner, id[:])
|
inner = protowire.AppendBytes(inner, id[:])
|
||||||
|
|
||||||
for i := range 36 {
|
for i := range 1000 {
|
||||||
path := strconv.FormatInt(int64(i), 36)
|
|
||||||
path = strings.Repeat("0", tt.pathLen-len(path)) + path
|
|
||||||
|
|
||||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||||
entry = protowire.AppendString(entry, path)
|
entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
|
||||||
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
||||||
entry = protowire.AppendBytes(entry, checksum)
|
entry = protowire.AppendBytes(entry, nil)
|
||||||
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
entry = protowire.AppendBytes(entry, nil)
|
||||||
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
||||||
@@ -306,16 +247,10 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
|
|||||||
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
||||||
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||||
entry = protowire.AppendString(entry, "a")
|
entry = protowire.AppendString(entry, "a")
|
||||||
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
||||||
entry = protowire.AppendBytes(entry, checksum)
|
entry = protowire.AppendBytes(entry, nil)
|
||||||
entry = append(entry, unknown...)
|
entry = append(entry, unknown...)
|
||||||
|
|
||||||
id := uuid.NewV4()
|
id := uuid.NewV4()
|
||||||
|
|||||||
Reference in new issue
Block a user