Default the manifest to index.mf and keep it out of its own listing (closes #100)
check / check (push) Failing after 2s
check / check (push) Failing after 2s
mfer gen now writes index.mf instead of .index.mf, the name fetch requests and the README calls the standard filename. Given a directory, check, freshen, list and export look only for index.mf; .index.mf is no longer recognized. Because index.mf is not hidden, the scanner now leaves the output file out of the listing by its path: gen --force over an existing manifest no longer lists the old one, while a file named index.mf in a subdirectory is still listed. Model: opus-5-5
This commit is contained in:
+1
-1
@@ -8,7 +8,7 @@ vendor.tzst
|
|||||||
modcache.tzst
|
modcache.tzst
|
||||||
|
|
||||||
# Generated manifest files
|
# Generated manifest files
|
||||||
.index.mf
|
/index.mf
|
||||||
|
|
||||||
# Secrets
|
# Secrets
|
||||||
.env
|
.env
|
||||||
|
|||||||
@@ -36,12 +36,12 @@ Generate a manifest for a directory tree, verify it later, and fetch a published
|
|||||||
tree by URL:
|
tree by URL:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
# Write .index.mf, a manifest of the files under the current directory.
|
# Write index.mf, a manifest of the files under the current directory.
|
||||||
bin/mfer gen .
|
bin/mfer gen .
|
||||||
|
|
||||||
# Verify the files on disk against the manifest. Exits nonzero if any file
|
# Verify the files on disk against the manifest. Exits nonzero if any file
|
||||||
# is missing or corrupted.
|
# is missing or corrupted.
|
||||||
bin/mfer check .index.mf
|
bin/mfer check index.mf
|
||||||
|
|
||||||
# Download and cryptographically verify a tree published over HTTP: mfer
|
# Download and cryptographically verify a tree published over HTTP: mfer
|
||||||
# fetches <url>/index.mf, then downloads every file it lists.
|
# fetches <url>/index.mf, then downloads every file it lists.
|
||||||
|
|||||||
+12
-15
@@ -75,25 +75,22 @@ func safeRateUint64(rate float64) uint64 {
|
|||||||
return uint64(rate)
|
return uint64(rate)
|
||||||
}
|
}
|
||||||
|
|
||||||
// findManifest looks for a manifest file in the given directory.
|
// findManifest returns the path of the manifest with the default name in
|
||||||
// It checks for index.mf and .index.mf, returning the first one found.
|
// dir, or an error if there is none.
|
||||||
func findManifest(fs afero.Fs, dir string) (string, error) {
|
func findManifest(fs afero.Fs, dir string) (string, error) {
|
||||||
candidates := []string{"index.mf", ".index.mf"}
|
path := filepath.Join(dir, defaultManifestName)
|
||||||
for _, name := range candidates {
|
|
||||||
path := filepath.Join(dir, name)
|
|
||||||
|
|
||||||
exists, err := afero.Exists(fs, path)
|
exists, err := afero.Exists(fs, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
|
||||||
|
|
||||||
if exists {
|
|
||||||
return path, nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return "", fmt.Errorf(
|
if !exists {
|
||||||
"%w in %s (looked for index.mf and .index.mf)", errNoManifestFound, dir)
|
return "", fmt.Errorf("%w in %s (looked for %s)",
|
||||||
|
errNoManifestFound, dir, defaultManifestName)
|
||||||
|
}
|
||||||
|
|
||||||
|
return path, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchManifestToTemp downloads a manifest URL to a temporary file and
|
// fetchManifestToTemp downloads a manifest URL to a temporary file and
|
||||||
|
|||||||
@@ -638,6 +638,43 @@ func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
|
|||||||
assert.Equal(t, "existing", string(content), "original file should be preserved")
|
assert.Equal(t, "existing", string(content), "original file should be preserved")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestGenerateLeavesOutputOutOfListing overwrites an output file inside the
|
||||||
|
// scanned tree with --force: the old file is not listed, while a file named
|
||||||
|
// index.mf in a subdirectory still is.
|
||||||
|
func TestGenerateLeavesOutputOutOfListing(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for name, output := range map[string]string{
|
||||||
|
"default name": "/testdir/index.mf",
|
||||||
|
"other name in a subdirectory": "/testdir/sub/listing.mf",
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewMemMapFs()
|
||||||
|
require.NoError(t, fs.MkdirAll("/testdir/sub", 0o755))
|
||||||
|
writeTestFile(t, fs, "/testdir/readme.txt", "hello")
|
||||||
|
writeTestFile(t, fs, "/testdir/sub/index.mf", "an ordinary file")
|
||||||
|
writeTestFile(t, fs, output, "previous manifest")
|
||||||
|
|
||||||
|
opts := testOpts([]string{
|
||||||
|
testApp, cmdGenerate, "-q", "--force", "-o", output, testDir,
|
||||||
|
}, fs)
|
||||||
|
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||||
|
|
||||||
|
manifest, err := mfer.NewManifestFromFile(fs, output)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
paths := make([]string, 0, len(manifest.Files()))
|
||||||
|
for _, f := range manifest.Files() {
|
||||||
|
paths = append(paths, f.GetPath())
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.ElementsMatch(t, []string{"readme.txt", "sub/index.mf"}, paths)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ func TestNoManifestFoundMessage(t *testing.T) {
|
|||||||
_, err := findManifest(afero.NewMemMapFs(), "/tmp/x")
|
_, err := findManifest(afero.NewMemMapFs(), "/tmp/x")
|
||||||
require.ErrorIs(t, err, errNoManifestFound)
|
require.ErrorIs(t, err, errNoManifestFound)
|
||||||
assert.EqualError(t, err,
|
assert.EqualError(t, err,
|
||||||
"no manifest found in /tmp/x (looked for index.mf and .index.mf)")
|
"no manifest found in /tmp/x (looked for index.mf)")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestVerifyRequiredSignerMessages(t *testing.T) {
|
func TestVerifyRequiredSignerMessages(t *testing.T) {
|
||||||
|
|||||||
@@ -313,7 +313,7 @@ func checkNoSymlinks(p string) error {
|
|||||||
|
|
||||||
// resolveManifestURL takes a URL and returns the manifest URL.
|
// resolveManifestURL takes a URL and returns the manifest URL.
|
||||||
// If the URL already ends with .mf, it's returned as-is.
|
// If the URL already ends with .mf, it's returned as-is.
|
||||||
// Otherwise, index.mf is appended.
|
// Otherwise, the default manifest name is appended.
|
||||||
func resolveManifestURL(inputURL string) (string, error) {
|
func resolveManifestURL(inputURL string) (string, error) {
|
||||||
parsed, err := url.Parse(inputURL)
|
parsed, err := url.Parse(inputURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -330,8 +330,7 @@ func resolveManifestURL(inputURL string) (string, error) {
|
|||||||
parsed.Path += "/"
|
parsed.Path += "/"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Append index.mf
|
parsed.Path += defaultManifestName
|
||||||
parsed.Path += "index.mf"
|
|
||||||
|
|
||||||
return parsed.String(), nil
|
return parsed.String(), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -157,7 +157,7 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Skip the manifest file itself
|
// Skip the manifest file itself
|
||||||
if relPath == s.manifestBase || relPath == "."+s.manifestBase {
|
if relPath == s.manifestBase {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
|||||||
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
||||||
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
||||||
Fs: mfa.Fs,
|
Fs: mfa.Fs,
|
||||||
|
OutputPath: ctx.String("output"),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set seed for deterministic UUID if provided
|
// Set seed for deterministic UUID if provided
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// resolveManifestArg resolves the manifest path from CLI arguments.
|
// resolveManifestArg resolves the manifest path from CLI arguments.
|
||||||
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf.
|
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf.
|
||||||
// If no argument is given, the current directory is searched.
|
// If no argument is given, the current directory is searched.
|
||||||
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
|
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
|
||||||
if ctx.Args().Len() > 0 {
|
if ctx.Args().Len() > 0 {
|
||||||
|
|||||||
@@ -24,6 +24,11 @@ const (
|
|||||||
flagProgress = "progress"
|
flagProgress = "progress"
|
||||||
|
|
||||||
manifestArgsUsage = "[manifest file]"
|
manifestArgsUsage = "[manifest file]"
|
||||||
|
|
||||||
|
// defaultManifestName is the filename gen writes by default, the one
|
||||||
|
// looked for when a command is given a directory, and the one fetch
|
||||||
|
// appends to a directory URL.
|
||||||
|
defaultManifestName = "index.mf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// errUnknownCommand indicates an unrecognized command argument.
|
// errUnknownCommand indicates an unrecognized command argument.
|
||||||
@@ -154,7 +159,7 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
|||||||
},
|
},
|
||||||
&cli.StringFlag{
|
&cli.StringFlag{
|
||||||
Name: "output",
|
Name: "output",
|
||||||
Value: "./.index.mf",
|
Value: defaultManifestName,
|
||||||
Aliases: []string{"o"},
|
Aliases: []string{"o"},
|
||||||
Usage: "Specify output filename",
|
Usage: "Specify output filename",
|
||||||
},
|
},
|
||||||
|
|||||||
+19
-1
@@ -57,6 +57,10 @@ type ScannerOptions struct {
|
|||||||
SigningOptions *SigningOptions
|
SigningOptions *SigningOptions
|
||||||
// Seed, if set, derives a deterministic UUID from this seed.
|
// Seed, if set, derives a deterministic UUID from this seed.
|
||||||
Seed string
|
Seed string
|
||||||
|
// OutputPath, if set, is the file the manifest will be written to.
|
||||||
|
// Enumeration skips that one file, so a manifest written inside the
|
||||||
|
// scanned tree, or one it replaces there, is never listed.
|
||||||
|
OutputPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
// FileEntry represents a file that has been enumerated.
|
// FileEntry represents a file that has been enumerated.
|
||||||
@@ -75,6 +79,7 @@ type Scanner struct {
|
|||||||
totalBytes FileSize // cached sum of all file sizes
|
totalBytes FileSize // cached sum of all file sizes
|
||||||
options *ScannerOptions
|
options *ScannerOptions
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
|
outputPath string // options.OutputPath made absolute; "" when unset
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewScanner creates a new Scanner with default options.
|
// NewScanner creates a new Scanner with default options.
|
||||||
@@ -93,11 +98,19 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
|
|||||||
fs = afero.NewOsFs()
|
fs = afero.NewOsFs()
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Scanner{
|
s := &Scanner{
|
||||||
files: make([]*FileEntry, 0),
|
files: make([]*FileEntry, 0),
|
||||||
options: opts,
|
options: opts,
|
||||||
fs: fs,
|
fs: fs,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if opts.OutputPath != "" {
|
||||||
|
// Abs fails only when there is no working directory, and then a
|
||||||
|
// relative output path could not be written either.
|
||||||
|
s.outputPath, _ = filepath.Abs(opts.OutputPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
|
// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
|
||||||
@@ -408,6 +421,11 @@ func (s *Scanner) enumerateFileWithInfo(
|
|||||||
// Compute absolute path for file reading
|
// Compute absolute path for file reading
|
||||||
absPath := filepath.Join(basePath, cleanPath)
|
absPath := filepath.Join(basePath, cleanPath)
|
||||||
|
|
||||||
|
// The manifest being written is not one of the files it lists.
|
||||||
|
if absPath == s.outputPath {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Handle symlinks
|
// Handle symlinks
|
||||||
if info.Mode()&fs.ModeSymlink != 0 {
|
if info.Mode()&fs.ModeSymlink != 0 {
|
||||||
if !s.options.FollowSymLinks {
|
if !s.options.FollowSymLinks {
|
||||||
|
|||||||
Reference in New Issue
Block a user