Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a7875cfe3f | ||
|
|
6a3f532c8d |
+24
-15
@@ -80,15 +80,15 @@ After decompressing `innerMessage`, the result is a serialized `MFFile`
|
||||
|
||||
Each file entry contains:
|
||||
|
||||
| Field | Number | Type | Description |
|
||||
| ---------- | ------ | ------------------------- | ------------------------------------------------------------- |
|
||||
| `path` | 1 | string | Relative file path (see Path Rules) |
|
||||
| `size` | 2 | int64 | File size in bytes |
|
||||
| `hashes` | 3 | repeated `MFFileChecksum` | At least one hash required; readers reject an entry with none |
|
||||
| `mimeType` | 301 | string (optional) | MIME type |
|
||||
| `mtime` | 302 | Timestamp (optional) | Modification time |
|
||||
| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) |
|
||||
| `mode` | 304 | uint32 | Permission bits (see File Mode) |
|
||||
| Field | Number | Type | Description |
|
||||
| ---------- | ------ | ------------------------- | ------------------------------------------------------------ |
|
||||
| `path` | 1 | string | Relative file path (see Path Rules) |
|
||||
| `size` | 2 | int64 | File size in bytes |
|
||||
| `hashes` | 3 | repeated `MFFileChecksum` | Must include exactly one SHA-256 multihash (see Hash Format) |
|
||||
| `mimeType` | 301 | string (optional) | MIME type |
|
||||
| `mtime` | 302 | Timestamp (optional) | Modification time |
|
||||
| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) |
|
||||
| `mode` | 304 | uint32 | Permission bits (see File Mode) |
|
||||
|
||||
## File Mode
|
||||
|
||||
@@ -133,8 +133,16 @@ Each checksum is a single `bytes multiHash` field containing a
|
||||
self-describing: the encoded bytes include a varint algorithm identifier
|
||||
followed by a varint digest length followed by the digest itself.
|
||||
|
||||
The 1.0 implementation writes SHA-256 multihashes (`0x12` algorithm code).
|
||||
Implementations must be able to verify SHA-256 multihashes at minimum.
|
||||
A writer must give every file entry exactly one SHA-256 multihash of the file's
|
||||
content: the algorithm code `0x12`, the digest length `0x20` and the 32 bytes of
|
||||
the digest. It may list multihashes of other algorithms beside it. A reader must
|
||||
refuse a manifest with an entry that has no SHA-256 multihash or more than one,
|
||||
whatever other hashes the entry lists. A verifier then needs only SHA-256,
|
||||
checking each file against its entry's SHA-256 multihash; it may ignore the
|
||||
other hashes.
|
||||
|
||||
The reference implementation writes one hash for each entry, the SHA-256
|
||||
multihash, and its `check` and `fetch` compute only SHA-256.
|
||||
|
||||
## Signature Scheme
|
||||
|
||||
@@ -212,10 +220,11 @@ compresses the same inner message to the same bytes:
|
||||
- Each entry holds only `path`, `size`, `hashes` with exactly one hash, `mtime`
|
||||
and `mode`. The hash is the SHA-256 multihash of the file's content, except
|
||||
that `freshen` reads only the files that are new or whose size, `mtime` or
|
||||
`mode` changed, and for every other file carries the first hash of its
|
||||
existing entry over unchanged. `mtime` is the file's modification time as the
|
||||
file system gives it, to the nanosecond, and `mode` is `0000` unless `gen` or
|
||||
`freshen` is given `--include-permissions`.
|
||||
`mode` changed, and for every other file carries the SHA-256 multihash of its
|
||||
existing entry over unchanged, leaving out any other hash the entry lists.
|
||||
`mtime` is the file's modification time as the file system gives it, to the
|
||||
nanosecond, and `mode` is `0000` unless `gen` or `freshen` is given
|
||||
`--include-permissions`.
|
||||
- `createdAt` is left out unless `gen` or `freshen` is given
|
||||
`--include-timestamps`. Then it holds the time the run started building the
|
||||
manifest, to the nanosecond, so it differs on every run.
|
||||
|
||||
@@ -126,8 +126,9 @@ type retryingClient struct {
|
||||
// to a limit that doubles each time, unless the server's Retry-After
|
||||
// header says how long to wait. Any other failure, an error from use
|
||||
// included, is returned at once and unwrapped; a non-OK status is
|
||||
// returned as errHTTPStatus. use must start over each time it is called,
|
||||
// since a retry calls it again with a new response.
|
||||
// returned as errHTTPStatus. If ctx ends before a retry, the error wraps
|
||||
// both ctx's error and the last try's error. use must start over each
|
||||
// time it is called, since a retry calls it again with a new response.
|
||||
func (c retryingClient) get(
|
||||
ctx context.Context, rawURL string, use func(*http.Response) error,
|
||||
) error {
|
||||
@@ -169,15 +170,19 @@ func (c retryingClient) get(
|
||||
}
|
||||
}
|
||||
|
||||
if !retry || attempt == fetchAttempts || ctx.Err() != nil {
|
||||
if !retry || attempt == fetchAttempts {
|
||||
return err
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return fmt.Errorf("%w: %w", ctx.Err(), err)
|
||||
}
|
||||
|
||||
log.Warnf("%s: %s, retrying in %s", rawURL, err, wait.Round(time.Millisecond))
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
return fmt.Errorf("%w: %w", ctx.Err(), err)
|
||||
case <-time.After(wait):
|
||||
}
|
||||
|
||||
|
||||
+46
-10
@@ -897,23 +897,59 @@ func TestGetHonorsRetryAfter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetStopsWaitingWhenCanceled gives get a server that always answers
|
||||
// 503 and an hour's wait before each retry, then lets the context expire
|
||||
// during that wait. get returning at all shows it stopped waiting.
|
||||
// instant503Transport answers every request with a 503 at once, so the
|
||||
// 503 is in hand before any deadline can pass.
|
||||
type instant503Transport struct{}
|
||||
|
||||
func (instant503Transport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
Body: http.NoBody,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TestGetStopsWaitingWhenCanceled answers every request with a 503 and
|
||||
// sets an hour's wait before each retry, then lets the context expire
|
||||
// during that wait. get returning at all shows it stopped waiting; it must
|
||||
// return the deadline error and keep the 503 with it.
|
||||
func TestGetStopsWaitingWhenCanceled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := testClient()
|
||||
client.client.Transport = instant503Transport{}
|
||||
client.firstDelay = time.Hour
|
||||
|
||||
err := getNothing(client, server.URL, 50*time.Millisecond)
|
||||
err := getNothing(client, "http://example.invalid/", 50*time.Millisecond)
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
require.ErrorIs(t, err, errHTTPStatus)
|
||||
}
|
||||
|
||||
// late503Transport answers every request with a 503, but only once the
|
||||
// request's context has ended. A real server cannot stand in for it: the
|
||||
// client drops a response still on its way when the context ends.
|
||||
type late503Transport struct{}
|
||||
|
||||
func (late503Transport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
<-req.Context().Done()
|
||||
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
Body: http.NoBody,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TestGetStopsWhenContextEndsDuringRequest lets the context expire while
|
||||
// the first request is still waiting for its 503. get must return the
|
||||
// deadline error and keep the 503 with it.
|
||||
func TestGetStopsWhenContextEndsDuringRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := testClient()
|
||||
client.client.Transport = late503Transport{}
|
||||
|
||||
err := getNothing(client, "http://example.invalid/", 50*time.Millisecond)
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
require.ErrorIs(t, err, errHTTPStatus)
|
||||
}
|
||||
|
||||
// TestDownloadFileRetriesToSuccess serves a file that fails twice before
|
||||
|
||||
+11
-11
@@ -33,9 +33,9 @@ var (
|
||||
// modification time where one is required to carry it forward unchanged.
|
||||
errEntryMissingMtime = errors.New("manifest entry has no mtime")
|
||||
|
||||
// errEntryMissingHash indicates a manifest entry that carries no hash
|
||||
// to carry forward.
|
||||
errEntryMissingHash = errors.New("manifest entry has no hash")
|
||||
// errEntryMissingHash indicates a manifest entry that carries no
|
||||
// SHA-256 hash to carry forward.
|
||||
errEntryMissingHash = errors.New("manifest entry has no SHA-256 hash")
|
||||
)
|
||||
|
||||
// errSignKeyNeeded indicates a signed manifest given to freshen with no
|
||||
@@ -699,26 +699,26 @@ func addFileToBuilder(
|
||||
// Entries reach this path only when recordEntry classified them as
|
||||
// unchanged, which requires a recorded mtime, so an absent mtime here is
|
||||
// an error rather than something to paper over with the Unix epoch.
|
||||
// Loading refuses an entry with no hash; one that got here anyway is an
|
||||
// error too, not a file to leave out of the manifest.
|
||||
// Only the entry's SHA-256 hash is carried over, wherever the entry lists
|
||||
// it, since check and fetch compare with no other. Loading refuses an
|
||||
// entry with no SHA-256 hash; one that got here anyway is an error too,
|
||||
// not a file to leave out of the manifest.
|
||||
func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
||||
mtime, ok := entryMtime(entry)
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath())
|
||||
}
|
||||
|
||||
if len(entry.GetHashes()) == 0 {
|
||||
hash := mfer.SHA256Multihash(entry)
|
||||
if hash == nil {
|
||||
return fmt.Errorf("%w: %s", errEntryMissingHash, entry.GetPath())
|
||||
}
|
||||
|
||||
err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
||||
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
||||
fs.FileMode(entry.GetMode()), entry.GetHashes()[0].GetMultiHash())
|
||||
fs.FileMode(entry.GetMode()), hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"manifest entry %s: %w (regenerate the manifest with mfer generate)",
|
||||
entry.GetPath(), err,
|
||||
)
|
||||
return fmt.Errorf("manifest entry %s: %w", entry.GetPath(), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -584,48 +587,76 @@ func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
|
||||
assert.Contains(t, err.Error(), "file1.txt")
|
||||
}
|
||||
|
||||
// TestFreshenAddExistingRejectsMissingHash pins that an entry with no hash
|
||||
// is an error rather than a file left out of the rebuilt manifest.
|
||||
// TestFreshenAddExistingRejectsMissingHash pins that an entry with no
|
||||
// SHA-256 hash, whether it lists no hash or only a SHA-1 one, is an error
|
||||
// rather than a file left out of the rebuilt manifest.
|
||||
func TestFreshenAddExistingRejectsMissingHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := mfer.NewBuilder()
|
||||
entry := &mfer.MFFilePath{
|
||||
Path: "data.bin",
|
||||
Size: 8,
|
||||
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
||||
}
|
||||
|
||||
err := addExistingToBuilder(b, entry)
|
||||
require.ErrorIs(t, err, errEntryMissingHash)
|
||||
assert.Contains(t, err.Error(), "data.bin")
|
||||
assert.Zero(t, b.FileCount())
|
||||
}
|
||||
|
||||
// TestFreshenAddExistingRejectsShortHash pins that an existing manifest
|
||||
// entry whose hash the builder refuses is reported as a problem with the
|
||||
// manifest, with the command that fixes it.
|
||||
func TestFreshenAddExistingRejectsShortHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
hashes []*mfer.MFFileChecksum
|
||||
}{
|
||||
{"no hash", nil},
|
||||
{"SHA-1 hash only", []*mfer.MFFileChecksum{{MultiHash: sha1Hash}}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := mfer.NewBuilder()
|
||||
entry := &mfer.MFFilePath{
|
||||
Path: "data.bin",
|
||||
Size: 8,
|
||||
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
||||
Hashes: tt.hashes,
|
||||
}
|
||||
|
||||
err := addExistingToBuilder(b, entry)
|
||||
require.ErrorIs(t, err, errEntryMissingHash)
|
||||
assert.Contains(t, err.Error(), "data.bin")
|
||||
assert.Zero(t, b.FileCount())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFreshenAddExistingKeepsSHA256Hash pins that an unchanged entry that
|
||||
// lists another hash before its SHA-256 hash is carried over with the
|
||||
// SHA-256 hash, the one check and fetch compare with, and nothing else.
|
||||
func TestFreshenAddExistingKeepsSHA256Hash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
|
||||
require.NoError(t, err)
|
||||
|
||||
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
b := mfer.NewBuilder()
|
||||
entry := &mfer.MFFilePath{
|
||||
Path: "old.txt",
|
||||
Size: 8,
|
||||
Path: "kept.txt",
|
||||
Size: 6,
|
||||
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
||||
Hashes: []*mfer.MFFileChecksum{
|
||||
{MultiHash: sha1Hash},
|
||||
{MultiHash: sha512Hash},
|
||||
{MultiHash: sha256Hash},
|
||||
},
|
||||
}
|
||||
|
||||
err = addExistingToBuilder(b, entry)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "manifest entry old.txt")
|
||||
assert.Contains(t, err.Error(), "mfer generate")
|
||||
assert.Zero(t, b.FileCount())
|
||||
require.NoError(t, addExistingToBuilder(b, entry))
|
||||
|
||||
var buf bytes.Buffer
|
||||
require.NoError(t, b.Build(context.Background(), &buf))
|
||||
|
||||
m, err := mfer.NewManifestFromReader(&buf)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, m.Files(), 1)
|
||||
require.Len(t, m.Files()[0].GetHashes(), 1)
|
||||
assert.Equal(t, sha256Hash, m.Files()[0].GetHashes()[0].GetMultiHash())
|
||||
}
|
||||
|
||||
// TestEntryMtime pins the presence semantics the callers depend on.
|
||||
|
||||
+26
-17
@@ -37,8 +37,7 @@ var (
|
||||
errPathDot = errors.New("contains '.' segment")
|
||||
errSizeMismatch = errors.New("size mismatch")
|
||||
errNegativeSize = errors.New("size cannot be negative")
|
||||
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
||||
errHashTooShort = errors.New("hash digest is too short")
|
||||
errHashNotSHA256 = errors.New("hash is not a SHA-256 multihash")
|
||||
errDuplicatePath = errors.New("duplicate path")
|
||||
)
|
||||
|
||||
@@ -113,6 +112,28 @@ func (m ModTime) Timestamp() *Timestamp {
|
||||
// Multihash represents a multihash-encoded file hash (typically SHA2-256).
|
||||
type Multihash []byte
|
||||
|
||||
// SHA256Multihash returns the first of entry's hashes that is a SHA-256
|
||||
// multihash, or nil if none is. Loading a manifest refuses an entry that
|
||||
// does not list exactly one, so for a loaded entry it is the only one.
|
||||
func SHA256Multihash(entry *MFFilePath) Multihash {
|
||||
for _, h := range entry.GetHashes() {
|
||||
if isSHA256Multihash(h.GetMultiHash()) {
|
||||
return h.GetMultiHash()
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isSHA256Multihash reports whether hash is a SHA-256 multihash: the
|
||||
// multihash code for SHA-256, then a 32-byte digest.
|
||||
func isSHA256Multihash(hash []byte) bool {
|
||||
decoded, err := multihash.Decode(hash)
|
||||
|
||||
return err == nil && decoded.Code == multihash.SHA2_256 &&
|
||||
len(decoded.Digest) == sha256.Size
|
||||
}
|
||||
|
||||
// FileHashProgress reports progress during file hashing.
|
||||
type FileHashProgress struct {
|
||||
BytesRead FileSize // Total bytes read so far for the current file
|
||||
@@ -241,8 +262,7 @@ func (b *Builder) FileCount() int {
|
||||
// This is useful when the hash is already known (e.g., from an existing manifest).
|
||||
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
||||
// Returns an error if path is invalid or already added, size is negative,
|
||||
// or hash is not a multihash with a digest of at least 32 bytes, as long
|
||||
// as SHA-256's.
|
||||
// or hash is not a SHA-256 multihash, the hash every entry must carry.
|
||||
func (b *Builder) AddFileWithHash(
|
||||
path RelFilePath,
|
||||
size FileSize,
|
||||
@@ -259,19 +279,8 @@ func (b *Builder) AddFileWithHash(
|
||||
return errNegativeSize
|
||||
}
|
||||
|
||||
decoded, err := multihash.Decode(hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", errHashNotMultihash, err)
|
||||
}
|
||||
|
||||
// The reader's limit on decoding cost (maxDecodedGrowth) assumes every
|
||||
// hash is at least as long as a SHA-256 multihash, so a manifest of
|
||||
// shorter ones could fail to load.
|
||||
if len(decoded.Digest) < sha256.Size {
|
||||
return fmt.Errorf(
|
||||
"%w: %d bytes, at least %d needed",
|
||||
errHashTooShort, len(decoded.Digest), sha256.Size,
|
||||
)
|
||||
if !isSHA256Multihash(hash) {
|
||||
return errHashNotSHA256
|
||||
}
|
||||
|
||||
entry := &MFFilePath{
|
||||
|
||||
+13
-10
@@ -5,6 +5,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -94,23 +95,25 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
||||
require.NoError(t, err)
|
||||
|
||||
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
|
||||
require.NoError(t, err)
|
||||
|
||||
truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
hash Multihash
|
||||
want error
|
||||
}{
|
||||
{"nil hash", nil, errHashNotMultihash},
|
||||
{"empty hash", []byte{}, errHashNotMultihash},
|
||||
{"one-byte hash", []byte{0x12}, errHashNotMultihash},
|
||||
{"nil hash", nil},
|
||||
{"empty hash", []byte{}},
|
||||
{"one-byte hash", []byte{0x12}},
|
||||
// A SHA-256 code and 32-byte length, then only two bytes of digest.
|
||||
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
|
||||
// A valid multihash, but its 20-byte SHA-1 digest is too short.
|
||||
{"SHA-1 multihash", sha1Hash, errHashTooShort},
|
||||
// A valid multihash whose 31-byte digest is one byte short.
|
||||
{"31-byte digest", truncatedHash, errHashTooShort},
|
||||
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}},
|
||||
{"SHA-1 multihash", sha1Hash},
|
||||
{"SHA-512 multihash", sha512Hash},
|
||||
// A SHA-256 code with a 31-byte digest.
|
||||
{"31-byte digest", truncatedHash},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -119,7 +122,7 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), 0, tt.hash)
|
||||
require.ErrorIs(t, err, tt.want)
|
||||
require.ErrorIs(t, err, errHashNotSHA256)
|
||||
assert.Equal(t, 0, b.FileCount())
|
||||
})
|
||||
}
|
||||
|
||||
+31
-5
@@ -27,7 +27,8 @@ var (
|
||||
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
||||
errInvalidFileFormat = errors.New("invalid file format")
|
||||
errInvalidManifestPath = errors.New("invalid file entry")
|
||||
errEntryMissingHash = errors.New("file entry has no hash")
|
||||
errEntryMissingHash = errors.New("file entry has no SHA-256 hash")
|
||||
errEntryExtraSHA256Hash = errors.New("file entry has more than one SHA-256 hash")
|
||||
errDecodedTooLarge = errors.New("too much memory needed")
|
||||
errSignerNotSigningKey = errors.New(
|
||||
"signer is not the fingerprint of the key that made the signature")
|
||||
@@ -236,6 +237,30 @@ func forEachBytesField(
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateEntryHashes refuses, naming its path, an entry that does not list
|
||||
// exactly one SHA-256 multihash; hashes of other algorithms beside it are
|
||||
// allowed. check and fetch compute only SHA-256, so with none they would
|
||||
// have nothing to check the file against, and with two check would pass a
|
||||
// file that matches either.
|
||||
func validateEntryHashes(entry *MFFilePath) error {
|
||||
sha256Hashes := 0
|
||||
|
||||
for _, h := range entry.GetHashes() {
|
||||
if isSHA256Multihash(h.GetMultiHash()) {
|
||||
sha256Hashes++
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case sha256Hashes == 0:
|
||||
return fmt.Errorf("%w: %q", errEntryMissingHash, entry.GetPath())
|
||||
case sha256Hashes > 1:
|
||||
return fmt.Errorf("%w: %q", errEntryExtraSHA256Hash, entry.GetPath())
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (m *manifest) deserializeInner() error {
|
||||
err := m.validateOuterHeader()
|
||||
if err != nil {
|
||||
@@ -294,8 +319,8 @@ func (m *manifest) deserializeInner() error {
|
||||
// untrusted .mf. Reject loudly on the first offender rather than
|
||||
// dropping entries, which would let a hostile manifest hide files from a
|
||||
// check. A path listed twice is refused too: check would check the one
|
||||
// file against both entries. So is an entry with no hash: there is
|
||||
// nothing to check the file against.
|
||||
// file against both entries. So is an entry that does not list exactly
|
||||
// one SHA-256 hash.
|
||||
seen := make(map[string]bool, len(m.pbInner.GetFiles()))
|
||||
|
||||
for _, f := range m.pbInner.GetFiles() {
|
||||
@@ -304,8 +329,9 @@ func (m *manifest) deserializeInner() error {
|
||||
return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
|
||||
}
|
||||
|
||||
if len(f.GetHashes()) == 0 {
|
||||
return fmt.Errorf("%w: %q", errEntryMissingHash, f.GetPath())
|
||||
err = validateEntryHashes(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if seen[f.GetPath()] {
|
||||
|
||||
@@ -5,8 +5,10 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"uuid"
|
||||
@@ -167,40 +169,94 @@ func TestDeserializeRefusesPathListedTwice(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest holding a file entry with no hash is refused as it is loaded,
|
||||
// naming the path: there is nothing to check the file against.
|
||||
func TestDeserializeRefusesEntryWithNoHash(t *testing.T) {
|
||||
// A file entry must list exactly one SHA-256 hash, and may list hashes of
|
||||
// other algorithms beside it. An entry with none, even one that lists a
|
||||
// SHA-512 hash, is refused as it is loaded, and so is an entry with two;
|
||||
// each refusal names the path. An entry listing a SHA-512 hash and then a
|
||||
// SHA-256 one loads with both.
|
||||
func TestDeserializeRequiresExactlyOneSHA256Hash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id := uuid.NewV4()
|
||||
inner, err := proto.Marshal(&MFFile{
|
||||
Version: MFFile_VERSION_ONE,
|
||||
Files: []*MFFilePath{{Path: "x", Size: 1}},
|
||||
Uuid: id[:],
|
||||
})
|
||||
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
||||
require.ErrorIs(t, err, errEntryMissingHash)
|
||||
require.EqualError(t, err, `file entry has no hash: "x"`)
|
||||
otherSHA256Hash, err := multihash.Encode(
|
||||
bytes.Repeat([]byte{1}, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
hashes [][]byte
|
||||
want error // nil for an entry that loads
|
||||
}{
|
||||
{"no hash", nil, errEntryMissingHash},
|
||||
{"SHA-512 hash only", [][]byte{sha512Hash}, errEntryMissingHash},
|
||||
{
|
||||
"two SHA-256 hashes",
|
||||
[][]byte{sha256Hash, otherSHA256Hash},
|
||||
errEntryExtraSHA256Hash,
|
||||
},
|
||||
{"SHA-512 hash then SHA-256 hash", [][]byte{sha512Hash, sha256Hash}, nil},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
entry := &MFFilePath{Path: "x", Size: 1}
|
||||
for _, h := range tt.hashes {
|
||||
entry.Hashes = append(entry.Hashes, &MFFileChecksum{MultiHash: h})
|
||||
}
|
||||
|
||||
id := uuid.NewV4()
|
||||
inner, err := proto.Marshal(&MFFile{
|
||||
Version: MFFile_VERSION_ONE,
|
||||
Files: []*MFFilePath{entry},
|
||||
Uuid: id[:],
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
||||
if tt.want == nil {
|
||||
require.NoError(t, err)
|
||||
require.Len(t, m.Files(), 1)
|
||||
assert.Len(t, m.Files()[0].GetHashes(), len(tt.hashes))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
require.ErrorIs(t, err, tt.want)
|
||||
assert.Contains(t, err.Error(), `"x"`)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
||||
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
|
||||
// and take 16 bytes plus the path to encode. A 37-character path makes that
|
||||
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
|
||||
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
|
||||
// it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
|
||||
// padded with zeros to that length, since a manifest lists a path only once.
|
||||
// Entries of a path, a SHA-256 hash, an empty MIME type and empty
|
||||
// modification and change times are counted at 432 bytes each (176 + 112 +
|
||||
// 16 + 64 + 64) and take 52 bytes plus the path to encode. A one-character
|
||||
// path makes that 53 bytes, about 8.2 times: refused, and leaving any one of
|
||||
// the five uncounted, even the MIME type, brings it under 8. A two-character
|
||||
// path makes it 54 bytes, 8 times exactly: loaded. There is one entry for
|
||||
// each base-36 digit, and each entry's path is its number in base 36, padded
|
||||
// with zeros to that length, since a manifest lists a path only once.
|
||||
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
pathLen int
|
||||
refused bool
|
||||
}{
|
||||
{37, true},
|
||||
{39, false},
|
||||
{1, true},
|
||||
{2, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -213,11 +269,14 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
||||
inner = protowire.AppendBytes(inner, id[:])
|
||||
|
||||
for i := range 1000 {
|
||||
for i := range 36 {
|
||||
path := strconv.FormatInt(int64(i), 36)
|
||||
path = strings.Repeat("0", tt.pathLen-len(path)) + path
|
||||
|
||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||
entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
|
||||
entry = protowire.AppendString(entry, path)
|
||||
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
entry = protowire.AppendBytes(entry, checksum)
|
||||
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
||||
@@ -247,10 +306,16 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
|
||||
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
||||
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
||||
|
||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
|
||||
require.NoError(t, err)
|
||||
|
||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||
entry = protowire.AppendString(entry, "a")
|
||||
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
entry = protowire.AppendBytes(entry, checksum)
|
||||
entry = append(entry, unknown...)
|
||||
|
||||
id := uuid.NewV4()
|
||||
|
||||
Reference in New Issue
Block a user