Default the manifest to index.mf and keep it out of its own listing (closes #100)
check / check (push) Failing after 1s
check / check (push) Failing after 1s
mfer gen now writes index.mf instead of .index.mf, the name fetch requests and the README calls the standard filename. Given a directory, check, freshen, list and export look only for index.mf; .index.mf is no longer recognized. Because index.mf is not hidden, gen and freshen leave out of their own listing the manifest they write and a temp file an interrupted run left beside it, matched by file identity (os.SameFile) however the path is spelled. The scanner takes these as a plain ExcludePaths list; manifestTempPath alone names the temp file, for both writes, both skips and the tests. Model: opus-5-5
This commit is contained in:
+12
-15
@@ -75,25 +75,22 @@ func safeRateUint64(rate float64) uint64 {
|
||||
return uint64(rate)
|
||||
}
|
||||
|
||||
// findManifest looks for a manifest file in the given directory.
|
||||
// It checks for index.mf and .index.mf, returning the first one found.
|
||||
// findManifest returns the path of the manifest with the default name in
|
||||
// dir, or an error if there is none.
|
||||
func findManifest(fs afero.Fs, dir string) (string, error) {
|
||||
candidates := []string{"index.mf", ".index.mf"}
|
||||
for _, name := range candidates {
|
||||
path := filepath.Join(dir, name)
|
||||
path := filepath.Join(dir, defaultManifestName)
|
||||
|
||||
exists, err := afero.Exists(fs, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if exists {
|
||||
return path, nil
|
||||
}
|
||||
exists, err := afero.Exists(fs, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return "", fmt.Errorf(
|
||||
"%w in %s (looked for index.mf and .index.mf)", errNoManifestFound, dir)
|
||||
if !exists {
|
||||
return "", fmt.Errorf("%w in %s (looked for %s)",
|
||||
errNoManifestFound, dir, defaultManifestName)
|
||||
}
|
||||
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// fetchManifestToTemp downloads a manifest URL to a temporary file and
|
||||
|
||||
+101
-5
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -27,7 +28,6 @@ const (
|
||||
testFile1 = "/testdir/file1.txt"
|
||||
testMF = "/testdir/test.mf"
|
||||
testOutput = "/output.mf"
|
||||
testOutputTmp = "/output.mf.tmp"
|
||||
testManifest = "/manifest.mf"
|
||||
testFlagBase = "--base"
|
||||
testFlagNoExtra = "--no-extra-files"
|
||||
@@ -682,7 +682,7 @@ func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
|
||||
assert.True(t, exists, "output file should exist")
|
||||
|
||||
// Verify temp file does NOT exist
|
||||
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
||||
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
||||
require.NoError(t, err)
|
||||
assert.False(t, tmpExists,
|
||||
"temp file should not exist after successful generation")
|
||||
@@ -714,7 +714,7 @@ func TestGenerateAtomicWriteOverwriteWithForce(t *testing.T) {
|
||||
"manifest should be overwritten")
|
||||
|
||||
// Verify temp file does NOT exist
|
||||
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
||||
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
||||
require.NoError(t, err)
|
||||
assert.False(t, tmpExists,
|
||||
"temp file should not exist after successful generation")
|
||||
@@ -743,6 +743,102 @@ func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
|
||||
assert.Equal(t, "existing", string(content), "original file should be preserved")
|
||||
}
|
||||
|
||||
// manifestPaths returns the file paths listed by the manifest at path.
|
||||
func manifestPaths(t *testing.T, fs afero.Fs, path string) []string {
|
||||
t.Helper()
|
||||
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: path,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
paths := make([]string, 0, len(manifest.Files()))
|
||||
for _, f := range manifest.Files() {
|
||||
paths = append(paths, f.GetPath())
|
||||
}
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
// TestGenerateLeavesOutputOutOfListing overwrites an output file inside the
|
||||
// scanned tree with --force: the old file is not listed, even when the tree
|
||||
// is named through a symlink, while a file named index.mf in a subdirectory
|
||||
// still is. The output file is recognized by file identity, which needs
|
||||
// the real filesystem.
|
||||
func TestGenerateLeavesOutputOutOfListing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Paths are relative to a temp dir holding data/tree and link, a
|
||||
// symlink to data.
|
||||
for name, tc := range map[string]struct{ input, output string }{
|
||||
"default name": {"data/tree", "data/tree/index.mf"},
|
||||
"other name in a subdirectory": {"data/tree", "data/tree/sub/listing.mf"},
|
||||
"tree named through a symlink": {"link/tree", "data/tree/index.mf"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
tree := filepath.Join(root, "data", "tree")
|
||||
output := filepath.Join(root, tc.output)
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
require.NoError(t, fs.MkdirAll(filepath.Join(tree, "sub"), 0o750))
|
||||
require.NoError(t,
|
||||
os.Symlink(filepath.Join(root, "data"), filepath.Join(root, "link")))
|
||||
writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
|
||||
writeTestFile(t, fs, filepath.Join(tree, "sub", "index.mf"), "an ordinary file")
|
||||
writeTestFile(t, fs, output, "previous manifest")
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdGenerate, "-q", "--force", "-o", output, filepath.Join(root, tc.input),
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.ElementsMatch(t, []string{testFileTxt, "sub/index.mf"},
|
||||
manifestPaths(t, fs, output))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGenerateDefaultOutputLeftOutOfListing runs gen with --force and no
|
||||
// other arguments, so it scans the current directory and writes the
|
||||
// relative path index.mf: the index.mf already there is not listed.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestGenerateDefaultOutputLeftOutOfListing(t *testing.T) {
|
||||
chdirTemp(t)
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
writeTestFile(t, fs, testFileTxt, "hello")
|
||||
writeTestFile(t, fs, "index.mf", "previous manifest")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "--force"}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, "index.mf"))
|
||||
}
|
||||
|
||||
// TestGenerateLeavesLeftoverTempFileOutOfListing runs gen where an
|
||||
// interrupted run left its temp file beside the output: the leftover is
|
||||
// not listed, and gen does not fail when it overwrites it.
|
||||
func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
output := filepath.Join(root, "index.mf")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
||||
writeTestFile(t, fs, manifestTempPath(output), "part of a manifest")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", output, root}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
|
||||
}
|
||||
|
||||
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -765,7 +861,7 @@ func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
||||
exists, _ := afero.Exists(fs, testOutput)
|
||||
assert.True(t, exists, "output file should exist")
|
||||
|
||||
tmpExists, _ := afero.Exists(fs, testOutputTmp)
|
||||
tmpExists, _ := afero.Exists(fs, manifestTempPath(testOutput))
|
||||
assert.False(t, tmpExists, "temp file should be cleaned up")
|
||||
|
||||
// Verify manifest is valid (not empty)
|
||||
@@ -831,7 +927,7 @@ func TestGenerateAtomicWriteCleansUpOnError(t *testing.T) {
|
||||
"output file should not exist after failed generation (atomic write)")
|
||||
|
||||
// Temp file should also not exist
|
||||
tmpExists, _ := afero.Exists(baseFs, testOutputTmp)
|
||||
tmpExists, _ := afero.Exists(baseFs, manifestTempPath(testOutput))
|
||||
assert.False(t, tmpExists,
|
||||
"temp file should be cleaned up after failed generation")
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ func TestNoManifestFoundMessage(t *testing.T) {
|
||||
_, err := findManifest(afero.NewMemMapFs(), "/tmp/x")
|
||||
require.ErrorIs(t, err, errNoManifestFound)
|
||||
assert.EqualError(t, err,
|
||||
"no manifest found in /tmp/x (looked for index.mf and .index.mf)")
|
||||
"no manifest found in /tmp/x (looked for index.mf)")
|
||||
}
|
||||
|
||||
func TestVerifyRequiredSignerMessages(t *testing.T) {
|
||||
|
||||
@@ -313,7 +313,7 @@ func checkNoSymlinks(p string) error {
|
||||
|
||||
// resolveManifestURL takes a URL and returns the manifest URL.
|
||||
// If the URL already ends with .mf, it's returned as-is.
|
||||
// Otherwise, index.mf is appended.
|
||||
// Otherwise, the default manifest name is appended.
|
||||
func resolveManifestURL(inputURL string) (string, error) {
|
||||
parsed, err := url.Parse(inputURL)
|
||||
if err != nil {
|
||||
@@ -330,8 +330,7 @@ func resolveManifestURL(inputURL string) (string, error) {
|
||||
parsed.Path += "/"
|
||||
}
|
||||
|
||||
// Append index.mf
|
||||
parsed.Path += "index.mf"
|
||||
parsed.Path += defaultManifestName
|
||||
|
||||
return parsed.String(), nil
|
||||
}
|
||||
|
||||
+22
-8
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
@@ -56,7 +57,7 @@ type freshenEntry struct {
|
||||
type freshenScanner struct {
|
||||
fs afero.Fs
|
||||
absBase string
|
||||
manifestBase string
|
||||
excluded []fs.FileInfo // files left out of the listing
|
||||
includeDotfiles bool
|
||||
followSymlinks bool
|
||||
showProgress bool
|
||||
@@ -156,11 +157,6 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
||||
"freshen: failed to compute relative path for %s: %w", path, err)
|
||||
}
|
||||
|
||||
// Skip the manifest file itself
|
||||
if relPath == s.manifestBase || relPath == "."+s.manifestBase {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handle dotfiles
|
||||
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
|
||||
if info.IsDir() {
|
||||
@@ -185,6 +181,12 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
||||
info = realInfo
|
||||
}
|
||||
|
||||
for _, excluded := range s.excluded {
|
||||
if os.SameFile(info, excluded) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
s.scanCount++
|
||||
|
||||
// Check against existing manifest
|
||||
@@ -307,7 +309,7 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
||||
func writeFreshenedManifest(
|
||||
ctx context.Context, afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
||||
) error {
|
||||
tmpPath := manifestPath + ".tmp"
|
||||
tmpPath := manifestTempPath(manifestPath)
|
||||
|
||||
outFile, err := afs.Create(tmpPath)
|
||||
if err != nil {
|
||||
@@ -364,10 +366,22 @@ func (mfa *CLIApp) freshenScan(
|
||||
startScan := time.Now()
|
||||
showProgress := ctx.Bool("progress")
|
||||
|
||||
// Leave out the manifest and a temp file left by an interrupted run,
|
||||
// as gen does. A path that cannot be stat'd, normally because no file
|
||||
// is there, needs no leaving out.
|
||||
var excluded []fs.FileInfo
|
||||
|
||||
for _, p := range []string{manifestPath, manifestTempPath(manifestPath)} {
|
||||
info, err := mfa.Fs.Stat(p)
|
||||
if err == nil {
|
||||
excluded = append(excluded, info)
|
||||
}
|
||||
}
|
||||
|
||||
scanner := &freshenScanner{
|
||||
fs: mfa.Fs,
|
||||
absBase: absBase,
|
||||
manifestBase: filepath.Base(manifestPath),
|
||||
excluded: excluded,
|
||||
includeDotfiles: ctx.Bool("include-dotfiles"),
|
||||
followSymlinks: ctx.Bool("follow-symlinks"),
|
||||
showProgress: showProgress,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -29,7 +30,7 @@ func (s stubFileInfo) IsDir() bool { return false }
|
||||
func (s stubFileInfo) Sys() any { return nil }
|
||||
|
||||
// setupFreshenDir populates /testdir with two files, scans it, and
|
||||
// writes the resulting manifest to /testdir/.index.mf.
|
||||
// writes the resulting manifest to /testdir/index.mf.
|
||||
func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
||||
t.Helper()
|
||||
|
||||
@@ -48,7 +49,7 @@ func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
||||
|
||||
// Write manifest to filesystem
|
||||
require.NoError(t,
|
||||
afero.WriteFile(fs, "/testdir/.index.mf", manifestBuf.Bytes(), 0o644))
|
||||
afero.WriteFile(fs, "/testdir/index.mf", manifestBuf.Bytes(), 0o644))
|
||||
}
|
||||
|
||||
func TestFreshenUnchanged(t *testing.T) {
|
||||
@@ -59,7 +60,7 @@ func TestFreshenUnchanged(t *testing.T) {
|
||||
|
||||
// Parse manifest to verify
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: "/testdir/.index.mf",
|
||||
Path: "/testdir/index.mf",
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -74,7 +75,7 @@ func TestFreshenWithChanges(t *testing.T) {
|
||||
|
||||
// Verify initial manifest has 2 files
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: "/testdir/.index.mf",
|
||||
Path: "/testdir/index.mf",
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -101,6 +102,64 @@ func TestFreshenWithChanges(t *testing.T) {
|
||||
assert.Equal(t, "modified content2", string(content))
|
||||
}
|
||||
|
||||
// TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a
|
||||
// subdirectory of the tree it lists: the manifest is not listed, while an
|
||||
// ordinary file of the same name at the top of the tree is. The manifest
|
||||
// is recognized by file identity, which needs the real filesystem.
|
||||
func TestFreshenLeavesManifestOutOfListing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
manifestPath := filepath.Join(root, "sub", "listing.mf")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
require.NoError(t, fs.MkdirAll(filepath.Join(root, "sub"), 0o750))
|
||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
||||
writeTestFile(t, fs, filepath.Join(root, "listing.mf"), "an ordinary file")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
// A new file gives freshen something to write.
|
||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.ElementsMatch(t, []string{testFileTxt, "listing.mf", "added.txt"},
|
||||
manifestPaths(t, fs, manifestPath))
|
||||
}
|
||||
|
||||
// TestFreshenLeavesLeftoverTempFileOutOfListing freshens a manifest where
|
||||
// an interrupted run left its temp file beside it: the leftover is not
|
||||
// listed.
|
||||
func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
manifestPath := filepath.Join(root, "index.mf")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
writeTestFile(t, fs, manifestTempPath(manifestPath), "part of a manifest")
|
||||
// A new file gives freshen something to write.
|
||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.ElementsMatch(t, []string{testFileTxt, "added.txt"},
|
||||
manifestPaths(t, fs, manifestPath))
|
||||
}
|
||||
|
||||
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
|
||||
// with respect to MFFilePath.Mtime, which is a message pointer with
|
||||
// proto3 field presence and may legitimately be absent.
|
||||
|
||||
+5
-1
@@ -89,11 +89,15 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
||||
|
||||
// buildScannerOptions constructs scanner options from the CLI flags.
|
||||
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
||||
output := ctx.String("output")
|
||||
opts := &mfer.ScannerOptions{
|
||||
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
||||
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
||||
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
||||
Fs: mfa.Fs,
|
||||
// Neither a manifest being replaced nor a temp file left by an
|
||||
// interrupted run belongs in the new manifest.
|
||||
ExcludePaths: []string{output, manifestTempPath(output)},
|
||||
}
|
||||
|
||||
// Set seed for deterministic UUID if provided
|
||||
@@ -217,7 +221,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
||||
}
|
||||
|
||||
// Create temp file for atomic write
|
||||
tmpPath := outputPath + ".tmp"
|
||||
tmpPath := manifestTempPath(outputPath)
|
||||
|
||||
outFile, err := mfa.Fs.Create(tmpPath)
|
||||
if err != nil {
|
||||
|
||||
@@ -65,7 +65,7 @@ func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
|
||||
}
|
||||
|
||||
// resolveManifestArg resolves the manifest path from CLI arguments.
|
||||
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf.
|
||||
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf.
|
||||
// If no argument is given, the current directory is searched.
|
||||
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
|
||||
if ctx.Args().Len() > 0 {
|
||||
|
||||
+12
-1
@@ -25,8 +25,19 @@ const (
|
||||
flagProgress = "progress"
|
||||
|
||||
manifestArgsUsage = "[manifest file]"
|
||||
|
||||
// defaultManifestName is the filename gen writes by default, the one
|
||||
// looked for when a command is given a directory, and the one fetch
|
||||
// appends to a directory URL.
|
||||
defaultManifestName = "index.mf"
|
||||
)
|
||||
|
||||
// manifestTempPath returns the temp file gen and freshen write a manifest
|
||||
// to before renaming it to out.
|
||||
func manifestTempPath(out string) string {
|
||||
return out + ".tmp"
|
||||
}
|
||||
|
||||
// errUnknownCommand indicates an unrecognized command argument.
|
||||
var errUnknownCommand = errors.New("unknown command")
|
||||
|
||||
@@ -156,7 +167,7 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "output",
|
||||
Value: "./.index.mf",
|
||||
Value: defaultManifestName,
|
||||
Aliases: []string{"o"},
|
||||
Usage: "Specify output filename",
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user