Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3e33c37c82 |
@@ -49,9 +49,7 @@ The `innerMessage` field is compressed with
|
||||
enforce a decompression size limit to prevent decompression bombs. The reference
|
||||
implementation limits decompressed size to 256 MB. It writes zstd frames with a
|
||||
window of at most 8 MiB, the largest window the zstd format recommends decoders
|
||||
support, and refuses frames that ask for a larger one. It also refuses an inner
|
||||
message whose file entries, hashes, timestamps and MIME types, counted at 160,
|
||||
112, 64 and 16 bytes each, add up to more than 8 times its size.
|
||||
support, and refuses frames that ask for a larger one.
|
||||
|
||||
## Inner Message (`MFFile`)
|
||||
|
||||
|
||||
@@ -12,13 +12,13 @@ require (
|
||||
github.com/pterm/pterm v0.12.35
|
||||
github.com/spf13/afero v1.8.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/urfave/cli/v2 v2.23.6
|
||||
github.com/urfave/cli/v2 v2.27.7
|
||||
google.golang.org/protobuf v1.28.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/atomicgo/cursor v0.0.1 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
|
||||
github.com/fatih/color v1.7.0 // indirect
|
||||
github.com/gookit/color v1.4.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
||||
@@ -34,7 +34,7 @@ require (
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
||||
golang.org/x/sys v0.1.0 // indirect
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||
|
||||
@@ -61,8 +61,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -231,12 +231,12 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
|
||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
||||
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
||||
github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
|
||||
github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
|
||||
github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
|
||||
github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
|
||||
+104
-5
@@ -8,6 +8,8 @@ import (
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
@@ -30,6 +32,7 @@ const (
|
||||
testFlagBase = "--base"
|
||||
testFlagNoExtra = "--no-extra-files"
|
||||
testFlagVersion = "--version"
|
||||
testFlagVerbose = "--verbose"
|
||||
)
|
||||
|
||||
var errSimulatedWrite = errors.New("simulated write failure")
|
||||
@@ -42,20 +45,32 @@ var errSimulatedWrite = errors.New("simulated write failure")
|
||||
var runMu sync.Mutex
|
||||
|
||||
// runCLI invokes RunWithOptions while holding runMu so parallel tests
|
||||
// capture their own output. Before releasing the lock it points the
|
||||
// process-global logger at io.Discard: other tests log outside the lock
|
||||
// (manifest loads, scans), and those lines must not land in this run's
|
||||
// buffers once it has returned and its test is reading them.
|
||||
// capture their own output, and returns its exit code.
|
||||
func runCLI(opts *RunOptions) int {
|
||||
exitCode, _ := runCLIWithLevel(opts)
|
||||
|
||||
return exitCode
|
||||
}
|
||||
|
||||
// runCLIWithLevel is runCLI that also returns the log level the run left
|
||||
// set, read while runMu still keeps other runs from changing it. Each run
|
||||
// starts at the default level, as a new process does. Before releasing the
|
||||
// lock it points the process-global logger at io.Discard: other tests log
|
||||
// outside the lock (manifest loads, scans), and those lines must not land in
|
||||
// this run's buffers once it has returned and its test is reading them.
|
||||
func runCLIWithLevel(opts *RunOptions) (int, log.Level) {
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
log.SetLevel(log.InfoLevel)
|
||||
|
||||
exitCode := RunWithOptions(opts)
|
||||
level := log.GetLevel()
|
||||
|
||||
log.SetOutput(io.Discard, io.Discard)
|
||||
log.Init()
|
||||
|
||||
return exitCode
|
||||
return exitCode, level
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
@@ -235,6 +250,90 @@ func TestRootVerbosityFlags(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// commandsTakingVerbose returns the command lines -v can follow: the root and
|
||||
// the generate, check, freshen and fetch subcommands.
|
||||
func commandsTakingVerbose() [][]string {
|
||||
return [][]string{
|
||||
{testApp},
|
||||
{testApp, cmdGenerate},
|
||||
{testApp, cmdCheck},
|
||||
{testApp, cmdFreshen},
|
||||
{testApp, cmdFetch},
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerboseCount asserts that one -v or --verbose gives verbose output and
|
||||
// two -v give debug output (issue #125). urfave/cli before v2.25.5 counted a
|
||||
// flag given by its alias twice, so one -v gave debug output.
|
||||
func TestVerboseCount(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
flags []string
|
||||
want log.Level
|
||||
}{
|
||||
{[]string{"-v"}, log.VerboseLevel},
|
||||
{[]string{testFlagVerbose}, log.VerboseLevel},
|
||||
{[]string{"-v", "-v"}, log.DebugLevel},
|
||||
}
|
||||
|
||||
for _, command := range commandsTakingVerbose() {
|
||||
for _, tc := range cases {
|
||||
args := slices.Concat(command, tc.flags)
|
||||
|
||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, level := runCLIWithLevel(testOpts(args, afero.NewMemMapFs()))
|
||||
assert.Equal(t, tc.want, level)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCombinedShortVerboseRefused asserts that -vv is refused (issue #125):
|
||||
// single-letter flags do not combine, so the -v help text says -v -v.
|
||||
func TestCombinedShortVerboseRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, command := range commandsTakingVerbose() {
|
||||
args := slices.Concat(command, []string{"-vv"})
|
||||
|
||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts(args, afero.NewMemMapFs())
|
||||
exitCode, level := runCLIWithLevel(opts)
|
||||
|
||||
assert.Equal(t, 1, exitCode)
|
||||
assert.Contains(t, testStderr(t, opts), "flag provided but not defined: -vv")
|
||||
assert.Equal(t, log.InfoLevel, level)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestShortAndLongVerboseRefused asserts that -v and --verbose given together
|
||||
// are refused (issue #125): urfave/cli v2 refuses a flag given under two of its
|
||||
// names, and one flag with an alias keeps help and parsing simple.
|
||||
func TestShortAndLongVerboseRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, command := range commandsTakingVerbose() {
|
||||
args := slices.Concat(command, []string{"-v", testFlagVerbose})
|
||||
|
||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts(args, afero.NewMemMapFs())
|
||||
exitCode, level := runCLIWithLevel(opts)
|
||||
|
||||
assert.Equal(t, 1, exitCode)
|
||||
assert.Contains(t, testStderr(t, opts), "Cannot use two forms of the same flag")
|
||||
assert.Equal(t, log.InfoLevel, level)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelpCommand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+10
-7
@@ -17,6 +17,7 @@ import (
|
||||
const (
|
||||
cmdGenerate = "generate"
|
||||
cmdCheck = "check"
|
||||
cmdFreshen = "freshen"
|
||||
cmdExport = "export"
|
||||
cmdFetch = "fetch"
|
||||
cmdVersion = "version"
|
||||
@@ -118,7 +119,7 @@ func commonFlags() []cli.Flag {
|
||||
&cli.BoolFlag{
|
||||
Name: "verbose",
|
||||
Aliases: []string{"v"},
|
||||
Usage: "Increase verbosity (-v for verbose, -vv for debug)",
|
||||
Usage: "Increase verbosity (-v for verbose, -v -v for debug)",
|
||||
Count: new(int),
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
@@ -131,9 +132,10 @@ func commonFlags() []cli.Flag {
|
||||
|
||||
func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdGenerate,
|
||||
Aliases: []string{"gen"},
|
||||
Usage: "Generate manifest file",
|
||||
Name: cmdGenerate,
|
||||
Aliases: []string{"gen"},
|
||||
Usage: "Generate manifest file",
|
||||
ArgsUsage: "[path ...]",
|
||||
Action: func(c *cli.Context) error {
|
||||
mfa.setVerbosity(c)
|
||||
mfa.printBanner()
|
||||
@@ -227,7 +229,7 @@ func (mfa *CLIApp) checkCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) freshenCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "freshen",
|
||||
Name: cmdFreshen,
|
||||
Usage: "Update manifest with changed, new, and removed files",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
Action: func(c *cli.Context) error {
|
||||
@@ -324,8 +326,9 @@ func (mfa *CLIApp) listCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) fetchCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdFetch,
|
||||
Usage: "fetch manifest and referenced files",
|
||||
Name: cmdFetch,
|
||||
Usage: "fetch manifest and referenced files",
|
||||
ArgsUsage: "URL",
|
||||
Action: func(c *cli.Context) error {
|
||||
mfa.setVerbosity(c)
|
||||
mfa.printBanner()
|
||||
|
||||
@@ -17,27 +17,4 @@ const (
|
||||
|
||||
// uuidLength is the length in bytes of a binary UUID.
|
||||
uuidLength = 16
|
||||
|
||||
// Numbers in mf.proto of MFFile.files and of the MFFilePath fields
|
||||
// that decoding sets aside a fixed amount of memory for.
|
||||
filesFieldNumber = 101
|
||||
hashesFieldNumber = 3
|
||||
mimeTypeFieldNumber = 301
|
||||
mtimeFieldNumber = 302
|
||||
ctimeFieldNumber = 303
|
||||
|
||||
// Bytes decoding sets aside for each file entry, hash, timestamp and
|
||||
// MIME type, however short its encoding. checkDecodedSize refuses an
|
||||
// inner message for which these add up to more than maxDecodedGrowth
|
||||
// times its size.
|
||||
decodedFileEntrySize = 160
|
||||
decodedHashSize = 112
|
||||
decodedTimestampSize = 64
|
||||
decodedMIMETypeSize = 16
|
||||
|
||||
// Each file entry mfer writes holds a path of at least one byte, a
|
||||
// 34-byte SHA-256 multihash and a modification time: at least 47 bytes,
|
||||
// counted at 336. So its manifests add up to at most about 7.15 times
|
||||
// their size, and this limit is about 12% above that.
|
||||
maxDecodedGrowth = 8
|
||||
)
|
||||
|
||||
+2
-91
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/spf13/afero"
|
||||
"google.golang.org/protobuf/encoding/protowire"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"sneak.berlin/go/mfer/internal/bork"
|
||||
"sneak.berlin/go/mfer/internal/log"
|
||||
@@ -28,8 +27,6 @@ var (
|
||||
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
||||
errInvalidFileFormat = errors.New("invalid file format")
|
||||
errInvalidManifestPath = errors.New("manifest contains invalid path")
|
||||
errDecodedTooLarge = errors.New(
|
||||
"manifest would take too much memory to decode")
|
||||
)
|
||||
|
||||
// validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
|
||||
@@ -157,85 +154,6 @@ func (m *manifest) decompressInner() ([]byte, error) {
|
||||
return dat, nil
|
||||
}
|
||||
|
||||
// checkDecodedSize refuses an encoded inner message whose file entries,
|
||||
// hashes, timestamps and MIME types would take more than maxDecodedGrowth
|
||||
// times its size to decode. Decoding sets aside a fixed amount for each,
|
||||
// however short its encoding, so a message of empty ones would take about
|
||||
// 50 times its size.
|
||||
func checkDecodedSize(inner []byte) error {
|
||||
limit := maxDecodedGrowth * int64(len(inner))
|
||||
|
||||
var decoded int64
|
||||
|
||||
add := func(size int64) error {
|
||||
decoded += size
|
||||
if decoded > limit {
|
||||
return errDecodedTooLarge
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return forEachBytesField(inner, func(num protowire.Number, entry []byte) error {
|
||||
if num != filesFieldNumber {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := add(decodedFileEntrySize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return forEachBytesField(entry, func(num protowire.Number, _ []byte) error {
|
||||
if num == hashesFieldNumber {
|
||||
return add(decodedHashSize)
|
||||
}
|
||||
|
||||
if num == mtimeFieldNumber || num == ctimeFieldNumber {
|
||||
return add(decodedTimestampSize)
|
||||
}
|
||||
|
||||
if num == mimeTypeFieldNumber {
|
||||
return add(decodedMIMETypeSize)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// forEachBytesField calls fn with the number and value of each
|
||||
// length-delimited field in the encoded message msg, and fails if msg is
|
||||
// malformed.
|
||||
func forEachBytesField(
|
||||
msg []byte, fn func(num protowire.Number, value []byte) error,
|
||||
) error {
|
||||
for len(msg) > 0 {
|
||||
num, wireType, tagLen := protowire.ConsumeTag(msg)
|
||||
if tagLen < 0 {
|
||||
return protowire.ParseError(tagLen)
|
||||
}
|
||||
|
||||
valueLen := protowire.ConsumeFieldValue(num, wireType, msg[tagLen:])
|
||||
if valueLen < 0 {
|
||||
return protowire.ParseError(valueLen)
|
||||
}
|
||||
|
||||
if wireType == protowire.BytesType {
|
||||
value, _ := protowire.ConsumeBytes(msg[tagLen:])
|
||||
|
||||
err := fn(num, value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
msg = msg[tagLen+valueLen:]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *manifest) deserializeInner() error {
|
||||
err := m.validateOuterHeader()
|
||||
if err != nil {
|
||||
@@ -259,16 +177,10 @@ func (m *manifest) deserializeInner() error {
|
||||
return bork.ErrFileTruncated
|
||||
}
|
||||
|
||||
err = checkDecodedSize(dat)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
|
||||
}
|
||||
|
||||
// Deserialize inner message
|
||||
m.pbInner = new(MFFile)
|
||||
|
||||
// Unknown fields would cost memory; mfer never writes a loaded manifest out.
|
||||
err = proto.UnmarshalOptions{DiscardUnknown: true}.Unmarshal(dat, m.pbInner)
|
||||
err = proto.Unmarshal(dat, m.pbInner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
|
||||
}
|
||||
@@ -337,8 +249,7 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
||||
// deserialize outer:
|
||||
m.pbOuter = new(MFFileOuter)
|
||||
|
||||
// Unknown fields would cost memory; mfer never writes a loaded manifest out.
|
||||
err = proto.UnmarshalOptions{DiscardUnknown: true}.Unmarshal(dat, m.pbOuter)
|
||||
err = proto.Unmarshal(dat, m.pbOuter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -54,15 +54,9 @@ func FuzzNewManifestFromReader(f *testing.F) {
|
||||
}
|
||||
|
||||
// It also keeps a few copies of its input. Buffers grow by
|
||||
// copying, so reaching those sizes allocates up to about six times
|
||||
// them in total. Decoding the decompressed data takes up to
|
||||
// maxDecodedGrowth times its size for file entries, hashes,
|
||||
// timestamps and MIME types, and drops fields it does not know.
|
||||
// The strings and bytes it copies out of it, such as many one-byte
|
||||
// values in one hash, take up to about five times more under the
|
||||
// race detector, which pads every small copy to 16 bytes, and about
|
||||
// half that without it. Twenty times the input and the
|
||||
// decompressed data leaves room for all of that.
|
||||
// copying, so reaching those sizes allocates a few times them in
|
||||
// total: sixteen times the input and the decompressed data leaves
|
||||
// room for that.
|
||||
//
|
||||
// The decoder also sets aside a new buffer of one to two times the
|
||||
// window for each frame that asks for a larger window than the
|
||||
@@ -77,9 +71,8 @@ func FuzzNewManifestFromReader(f *testing.F) {
|
||||
// fails if the decoder accepts windows of twice zstdWindowSize; the
|
||||
// seed whose two frames together exceed MaxDecompressedSize fails
|
||||
// if the decoder decodes them in full instead of stopping at the
|
||||
// declared size; the seeds of empty file entries and of a file
|
||||
// entry of empty hashes fail if the parser decodes them.
|
||||
limit := 20*(uint64(len(data))+decompressed) + 24*zstdWindowSize
|
||||
// declared size.
|
||||
limit := 16*(uint64(len(data))+decompressed) + 24*zstdWindowSize
|
||||
|
||||
allocated := after.TotalAlloc - before.TotalAlloc
|
||||
if allocated > limit {
|
||||
|
||||
@@ -6,10 +6,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
@@ -117,113 +114,6 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
||||
// and change times are counted at 416 bytes each (160 + 112 + 16 + 64 + 64)
|
||||
// and take 16 bytes plus the path to encode. A 35-character path makes that
|
||||
// 51 bytes, about 8.2 times: refused, and leaving any one of the five
|
||||
// uncounted, even the MIME type, brings it under 8. A 37-character path makes
|
||||
// it 53 bytes, about 7.8 times: loaded.
|
||||
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
pathLen int
|
||||
refused bool
|
||||
}{
|
||||
{35, true},
|
||||
{37, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||
entry = protowire.AppendString(entry, strings.Repeat("a", tt.pathLen))
|
||||
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
|
||||
id := uuid.New()
|
||||
inner := protowire.AppendTag(nil, 102, protowire.BytesType) // MFFile.uuid
|
||||
inner = protowire.AppendBytes(inner, id[:])
|
||||
|
||||
for range 1000 {
|
||||
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
||||
inner = protowire.AppendBytes(inner, entry)
|
||||
}
|
||||
|
||||
_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
||||
if tt.refused {
|
||||
require.ErrorIs(t, err, errDecodedTooLarge)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Fields the decoder does not know are dropped, in the outer message, the inner
|
||||
// message and a file entry, so that they take no memory once loaded.
|
||||
func TestDeserializeDropsUnknownFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
||||
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
||||
|
||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||
entry = protowire.AppendString(entry, "a")
|
||||
entry = append(entry, unknown...)
|
||||
|
||||
id := uuid.New()
|
||||
inner := protowire.AppendTag(nil, 101, protowire.BytesType) // MFFile.files
|
||||
inner = protowire.AppendBytes(inner, entry)
|
||||
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
||||
inner = protowire.AppendBytes(inner, id[:])
|
||||
inner = append(inner, unknown...)
|
||||
|
||||
data := wrapInner(t, id, inner)
|
||||
data = append(data, unknown...) // the outer message ends the file
|
||||
|
||||
m, err := NewManifestFromReader(bytes.NewReader(data))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, m.Files(), 1)
|
||||
assert.Empty(t, m.pbOuter.ProtoReflect().GetUnknown())
|
||||
assert.Empty(t, m.pbInner.ProtoReflect().GetUnknown())
|
||||
assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
|
||||
}
|
||||
|
||||
// Many empty files with names of at most three characters and modification
|
||||
// times at the epoch make about the densest manifest mfer writes: it takes
|
||||
// about 7 times its size to decode, and still loads. A signature would not
|
||||
// change the inner message, so none is added.
|
||||
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
||||
|
||||
b := NewBuilder()
|
||||
b.SetIncludeTimestamps(true)
|
||||
|
||||
const files = 10000
|
||||
for i := range files {
|
||||
name := RelFilePath(strconv.FormatInt(int64(i), 36))
|
||||
require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), hash))
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
require.NoError(t, b.Build(context.Background(), &buf))
|
||||
|
||||
m, err := NewManifestFromReader(&buf)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, m.Files(), files)
|
||||
}
|
||||
|
||||
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
go test fuzz v1
|
||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06\xff\xff\xff\x03\xc2\x06 {\x16\xbdu\xa0\xa2\x11\xfcH\xef*\x1b7\r\x99\xefb\x04\x02g\n\xa9\xf3B5\xe5p\x96\x8c\x8c\xac\x0e\xca\x06\x10\x03Q\xb2\xd0\x19`F\xc1\xb1\xc0Z\xf4x\xf4g^\xba\f\xa1\x06(\xb5/\xfd\x04h\x04\x01\x00d\x01\xb2\x06\x10\x03Q\xb2\xd0\x19`F\xc1\xb1\xc0Z\xf4x\xf4g^\xaa\x06\x00\x01T\x13\x024\xce\xff\rL\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15M\x00\x00\x00\x01T\x00\x044\xfc\xff\x153\xea\a\xb4")
|
||||
@@ -1,2 +0,0 @@
|
||||
go test fuzz v1
|
||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06\xff\xff\xff\x03\xc2\x06 .\xcd\x11|0\xfcP\xe5\x1b\xe3\xc6Ӡ\xcdڤx\xcd\x169t\x1a9~ǽB\xc9\xe8G`\x05\xca\x06\x10\x11*!\x0e\x95EF\xb8\xbd\x9f\xde\x12MF\r\x99\xba\f\xa6\x06(\xb5/\xfd\x04h,\x01\x00\xb4\x01\xb2\x06\x10\x11*!\x0e\x95EF\xb8\xbd\x9f\xde\x12MF\r\x99\xaa\x06\xe6\xff\xff\x03\x1a\x00\x01T\x14\x024\x8b\xff\x17L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15M\x00\x00\x00\x01T\x00\x044\xfc\xff\x15\x02\xd1.\xe3")
|
||||
Reference in New Issue
Block a user