Compare commits

...
2 Commits
Author SHA1 Message Date
sneak 97875a82bc Default the manifest to index.mf and keep it out of its own listing (closes #100)
check / check (push) Failing after 1s
mfer gen now writes index.mf instead of .index.mf, the name fetch
requests and the README calls the standard filename. Given a
directory, check, freshen, list and export look only for index.mf;
.index.mf is no longer recognized. Because index.mf is not hidden, gen
and freshen now leave out of their own listing both the manifest they
write and a temp file an interrupted run left beside it, by file
identity (os.SameFile), so each is recognized however its path is
spelled, while an ordinary file of the same name elsewhere in the tree
is still listed.

Model: opus-5-5
2026-10-04 10:54:31 +00:00
clawbot 0a9963002c NewChecker takes CheckerOptions instead of positional arguments (closes #78)
check / check (push) Failing after 1s
NewChecker now takes *CheckerOptions (ManifestPath, BasePath, Fs), named
like ScannerOptions. A nil Fs still means the OS filesystem, as before and
as in ScannerOptions; nil options or an empty path return an error naming
the missing path.

Audit of the other exported constructors in mfer: NewManifestFromFile took
a filesystem and a path positionally; it now takes
*ManifestFromFileOptions (Path, Fs) with the same nil and empty rules.
NewBuilder and NewScanner take no arguments, NewScannerWithOptions already
takes options, and NewManifestFromReader takes one reader, which the style
guide exempts; these are unchanged.

Model: opus-5-5
2026-10-04 12:19:31 +02:00
16 changed files with 555 additions and 105 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ vendor.tzst
modcache.tzst modcache.tzst
# Generated manifest files # Generated manifest files
.index.mf /index.mf
# Secrets # Secrets
.env .env
+2 -2
View File
@@ -36,12 +36,12 @@ Generate a manifest for a directory tree, verify it later, and fetch a published
tree by URL: tree by URL:
```sh ```sh
# Write .index.mf, a manifest of the files under the current directory. # Write index.mf, a manifest of the files under the current directory.
bin/mfer gen . bin/mfer gen .
# Verify the files on disk against the manifest. Exits nonzero if any file # Verify the files on disk against the manifest. Exits nonzero if any file
# is missing or corrupted. # is missing or corrupted.
bin/mfer check .index.mf bin/mfer check index.mf
# Download and cryptographically verify a tree published over HTTP: mfer # Download and cryptographically verify a tree published over HTTP: mfer
# fetches <url>/index.mf, then downloads every file it lists. # fetches <url>/index.mf, then downloads every file it lists.
+17 -16
View File
@@ -75,25 +75,22 @@ func safeRateUint64(rate float64) uint64 {
return uint64(rate) return uint64(rate)
} }
// findManifest looks for a manifest file in the given directory. // findManifest returns the path of the manifest with the default name in
// It checks for index.mf and .index.mf, returning the first one found. // dir, or an error if there is none.
func findManifest(fs afero.Fs, dir string) (string, error) { func findManifest(fs afero.Fs, dir string) (string, error) {
candidates := []string{"index.mf", ".index.mf"} path := filepath.Join(dir, defaultManifestName)
for _, name := range candidates {
path := filepath.Join(dir, name)
exists, err := afero.Exists(fs, path) exists, err := afero.Exists(fs, path)
if err != nil { if err != nil {
return "", err return "", err
}
if exists {
return path, nil
}
} }
return "", fmt.Errorf( if !exists {
"%w in %s (looked for index.mf and .index.mf)", errNoManifestFound, dir) return "", fmt.Errorf("%w in %s (looked for %s)",
errNoManifestFound, dir, defaultManifestName)
}
return path, nil
} }
// fetchManifestToTemp downloads a manifest URL to a temporary file and // fetchManifestToTemp downloads a manifest URL to a temporary file and
@@ -298,7 +295,11 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
log.Infof("checking manifest %s with base %s", manifestPath, basePath) log.Infof("checking manifest %s with base %s", manifestPath, basePath)
// Create checker // Create checker
chk, err := mfer.NewChecker(manifestPath, basePath, mfa.Fs) chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestPath: manifestPath,
BasePath: basePath,
Fs: mfa.Fs,
})
if err != nil { if err != nil {
return fmt.Errorf("failed to load manifest: %w", err) return fmt.Errorf("failed to load manifest: %w", err)
} }
+106 -2
View File
@@ -8,6 +8,7 @@ import (
"io" "io"
"math/rand" "math/rand"
"os" "os"
"path/filepath"
"slices" "slices"
"strings" "strings"
"sync" "sync"
@@ -504,7 +505,10 @@ func TestGenerateExcludesDotfilesByDefault(t *testing.T) {
assert.True(t, exists) assert.True(t, exists)
// Verify manifest only has 1 file (the non-dotfile) // Verify manifest only has 1 file (the non-dotfile)
manifest, err := mfer.NewManifestFromFile(fs, testMF) manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: testMF,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 1) assert.Len(t, manifest.Files(), 1)
assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath()) assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath())
@@ -528,7 +532,10 @@ func TestGenerateWithIncludeDotfiles(t *testing.T) {
require.Equal(t, 0, exitCode) require.Equal(t, 0, exitCode)
// Verify manifest has 2 files (including dotfile) // Verify manifest has 2 files (including dotfile)
manifest, err := mfer.NewManifestFromFile(fs, testMF) manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: testMF,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 2) assert.Len(t, manifest.Files(), 2)
} }
@@ -737,6 +744,103 @@ func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
assert.Equal(t, "existing", string(content), "original file should be preserved") assert.Equal(t, "existing", string(content), "original file should be preserved")
} }
// manifestPaths returns the file paths listed by the manifest at path.
func manifestPaths(t *testing.T, fs afero.Fs, path string) []string {
t.Helper()
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: path,
Fs: fs,
})
require.NoError(t, err)
paths := make([]string, 0, len(manifest.Files()))
for _, f := range manifest.Files() {
paths = append(paths, f.GetPath())
}
return paths
}
// TestGenerateLeavesOutputOutOfListing overwrites an output file inside the
// scanned tree with --force: the old file is not listed, even when the tree
// is named through a symlink, while a file named index.mf in a subdirectory
// still is. The output file is recognized by file identity, which needs
// the real filesystem.
func TestGenerateLeavesOutputOutOfListing(t *testing.T) {
t.Parallel()
// Paths are relative to a temp dir holding data/tree and link, a
// symlink to data.
for name, tc := range map[string]struct{ input, output string }{
"default name": {"data/tree", "data/tree/index.mf"},
"other name in a subdirectory": {"data/tree", "data/tree/sub/listing.mf"},
"tree named through a symlink": {"link/tree", "data/tree/index.mf"},
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
root := t.TempDir()
tree := filepath.Join(root, "data", "tree")
output := filepath.Join(root, tc.output)
fs := afero.NewOsFs()
require.NoError(t, fs.MkdirAll(filepath.Join(tree, "sub"), 0o750))
require.NoError(t,
os.Symlink(filepath.Join(root, "data"), filepath.Join(root, "link")))
writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
writeTestFile(t, fs, filepath.Join(tree, "sub", "index.mf"), "an ordinary file")
writeTestFile(t, fs, output, "previous manifest")
opts := testOpts([]string{
testApp, cmdGenerate, "-q", "--force", "-o", output, filepath.Join(root, tc.input),
}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
assert.ElementsMatch(t, []string{testFileTxt, "sub/index.mf"},
manifestPaths(t, fs, output))
})
}
}
// TestGenerateDefaultOutputLeftOutOfListing runs gen with --force and no
// other arguments, so it scans the current directory and writes the
// relative path index.mf: the index.mf already there is not listed.
//
//nolint:paralleltest // changes the process-global working directory
func TestGenerateDefaultOutputLeftOutOfListing(t *testing.T) {
chdirTemp(t)
fs := afero.NewOsFs()
writeTestFile(t, fs, testFileTxt, "hello")
writeTestFile(t, fs, "index.mf", "previous manifest")
opts := testOpts([]string{testApp, cmdGenerate, "-q", "--force"}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, "index.mf"))
}
// TestGenerateLeavesLeftoverTempFileOutOfListing runs gen where an
// interrupted run left its temp file, index.mf.tmp, beside the output:
// the leftover is not listed, and gen does not fail when it overwrites
// it.
func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
t.Parallel()
root := t.TempDir()
output := filepath.Join(root, "index.mf")
fs := afero.NewOsFs()
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
writeTestFile(t, fs, output+".tmp", "part of a manifest")
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", output, root}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
}
func TestGenerateAtomicWriteUsesTemp(t *testing.T) { func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
t.Parallel() t.Parallel()
+11 -3
View File
@@ -61,7 +61,11 @@ func unsignedChecker(t *testing.T) *mfer.Checker {
require.NoError(t, s.ToManifest(context.Background(), &buf, nil)) require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644)) require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644))
chk, err := mfer.NewChecker("/d/index.mf", "/d", fs) chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestPath: "/d/index.mf",
BasePath: "/d",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
require.False(t, chk.IsSigned()) require.False(t, chk.IsSigned())
@@ -74,7 +78,7 @@ func TestNoManifestFoundMessage(t *testing.T) {
_, err := findManifest(afero.NewMemMapFs(), "/tmp/x") _, err := findManifest(afero.NewMemMapFs(), "/tmp/x")
require.ErrorIs(t, err, errNoManifestFound) require.ErrorIs(t, err, errNoManifestFound)
assert.EqualError(t, err, assert.EqualError(t, err,
"no manifest found in /tmp/x (looked for index.mf and .index.mf)") "no manifest found in /tmp/x (looked for index.mf)")
} }
func TestVerifyRequiredSignerMessages(t *testing.T) { func TestVerifyRequiredSignerMessages(t *testing.T) {
@@ -167,7 +171,11 @@ func signedChecker(t *testing.T) *mfer.Checker {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/index.mf", buf.Bytes(), 0o644)) require.NoError(t, afero.WriteFile(fs, "/index.mf", buf.Bytes(), 0o644))
chk, err := mfer.NewChecker("/index.mf", "/", fs) chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestPath: "/index.mf",
BasePath: "/",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
require.True(t, chk.IsSigned()) require.True(t, chk.IsSigned())
+2 -3
View File
@@ -313,7 +313,7 @@ func checkNoSymlinks(p string) error {
// resolveManifestURL takes a URL and returns the manifest URL. // resolveManifestURL takes a URL and returns the manifest URL.
// If the URL already ends with .mf, it's returned as-is. // If the URL already ends with .mf, it's returned as-is.
// Otherwise, index.mf is appended. // Otherwise, the default manifest name is appended.
func resolveManifestURL(inputURL string) (string, error) { func resolveManifestURL(inputURL string) (string, error) {
parsed, err := url.Parse(inputURL) parsed, err := url.Parse(inputURL)
if err != nil { if err != nil {
@@ -330,8 +330,7 @@ func resolveManifestURL(inputURL string) (string, error) {
parsed.Path += "/" parsed.Path += "/"
} }
// Append index.mf parsed.Path += defaultManifestName
parsed.Path += "index.mf"
return parsed.String(), nil return parsed.String(), nil
} }
+32 -9
View File
@@ -7,6 +7,7 @@ import (
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
"os"
"path/filepath" "path/filepath"
"time" "time"
@@ -56,7 +57,8 @@ type freshenEntry struct {
type freshenScanner struct { type freshenScanner struct {
fs afero.Fs fs afero.Fs
absBase string absBase string
manifestBase string manifestInfo fs.FileInfo // the manifest being freshened
tmpInfo fs.FileInfo // its leftover temp file; nil when there is none
includeDotfiles bool includeDotfiles bool
followSymlinks bool followSymlinks bool
showProgress bool showProgress bool
@@ -156,11 +158,6 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
"freshen: failed to compute relative path for %s: %w", path, err) "freshen: failed to compute relative path for %s: %w", path, err)
} }
// Skip the manifest file itself
if relPath == s.manifestBase || relPath == "."+s.manifestBase {
return nil
}
// Handle dotfiles // Handle dotfiles
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) { if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
if info.IsDir() { if info.IsDir() {
@@ -185,6 +182,13 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
info = realInfo info = realInfo
} }
// Skip the manifest file itself and a temp file left beside it by an
// interrupted run, however their paths are spelled; gen leaves out its
// output file the same way. os.SameFile never matches a nil FileInfo.
if os.SameFile(info, s.manifestInfo) || os.SameFile(info, s.tmpInfo) {
return nil
}
s.scanCount++ s.scanCount++
// Check against existing manifest // Check against existing manifest
@@ -364,17 +368,33 @@ func (mfa *CLIApp) freshenScan(
startScan := time.Now() startScan := time.Now()
showProgress := ctx.Bool("progress") showProgress := ctx.Bool("progress")
manifestInfo, err := mfa.Fs.Stat(manifestPath)
if err != nil {
return nil, 0, fmt.Errorf("freshen: %w", err)
}
// The temp file writeFreshenedManifest writes, if an interrupted run
// left one. One that cannot be stat'd, normally because it does not
// exist, needs no leaving out.
var tmpInfo fs.FileInfo
info, err := mfa.Fs.Stat(manifestPath + ".tmp")
if err == nil {
tmpInfo = info
}
scanner := &freshenScanner{ scanner := &freshenScanner{
fs: mfa.Fs, fs: mfa.Fs,
absBase: absBase, absBase: absBase,
manifestBase: filepath.Base(manifestPath), manifestInfo: manifestInfo,
tmpInfo: tmpInfo,
includeDotfiles: ctx.Bool("include-dotfiles"), includeDotfiles: ctx.Bool("include-dotfiles"),
followSymlinks: ctx.Bool("follow-symlinks"), followSymlinks: ctx.Bool("follow-symlinks"),
showProgress: showProgress, showProgress: showProgress,
existingByPath: existingByPath, existingByPath: existingByPath,
} }
err := afero.Walk(mfa.Fs, absBase, scanner.walk) err = afero.Walk(mfa.Fs, absBase, scanner.walk)
if showProgress { if showProgress {
log.ProgressDone() log.ProgressDone()
@@ -445,7 +465,10 @@ func (mfa *CLIApp) loadExistingEntries(
log.Infof("loading manifest from %s", manifestPath) log.Infof("loading manifest from %s", manifestPath)
// Load existing manifest // Load existing manifest
manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath) manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: manifestPath,
Fs: mfa.Fs,
})
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to load manifest: %w", err) return nil, fmt.Errorf("failed to load manifest: %w", err)
} }
+69 -4
View File
@@ -5,6 +5,7 @@ import (
"bytes" "bytes"
"context" "context"
"os" "os"
"path/filepath"
"testing" "testing"
"time" "time"
@@ -29,7 +30,7 @@ func (s stubFileInfo) IsDir() bool { return false }
func (s stubFileInfo) Sys() any { return nil } func (s stubFileInfo) Sys() any { return nil }
// setupFreshenDir populates /testdir with two files, scans it, and // setupFreshenDir populates /testdir with two files, scans it, and
// writes the resulting manifest to /testdir/.index.mf. // writes the resulting manifest to /testdir/index.mf.
func setupFreshenDir(t *testing.T, fs afero.Fs) { func setupFreshenDir(t *testing.T, fs afero.Fs) {
t.Helper() t.Helper()
@@ -48,7 +49,7 @@ func setupFreshenDir(t *testing.T, fs afero.Fs) {
// Write manifest to filesystem // Write manifest to filesystem
require.NoError(t, require.NoError(t,
afero.WriteFile(fs, "/testdir/.index.mf", manifestBuf.Bytes(), 0o644)) afero.WriteFile(fs, "/testdir/index.mf", manifestBuf.Bytes(), 0o644))
} }
func TestFreshenUnchanged(t *testing.T) { func TestFreshenUnchanged(t *testing.T) {
@@ -58,7 +59,10 @@ func TestFreshenUnchanged(t *testing.T) {
setupFreshenDir(t, fs) setupFreshenDir(t, fs)
// Parse manifest to verify // Parse manifest to verify
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf") manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: "/testdir/index.mf",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 2) assert.Len(t, manifest.Files(), 2)
} }
@@ -70,7 +74,10 @@ func TestFreshenWithChanges(t *testing.T) {
setupFreshenDir(t, fs) setupFreshenDir(t, fs)
// Verify initial manifest has 2 files // Verify initial manifest has 2 files
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf") manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: "/testdir/index.mf",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 2) assert.Len(t, manifest.Files(), 2)
@@ -95,6 +102,64 @@ func TestFreshenWithChanges(t *testing.T) {
assert.Equal(t, "modified content2", string(content)) assert.Equal(t, "modified content2", string(content))
} }
// TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a
// subdirectory of the tree it lists: the manifest is not listed, while an
// ordinary file of the same name at the top of the tree is. The manifest
// is recognized by file identity, which needs the real filesystem.
func TestFreshenLeavesManifestOutOfListing(t *testing.T) {
t.Parallel()
root := t.TempDir()
manifestPath := filepath.Join(root, "sub", "listing.mf")
fs := afero.NewOsFs()
require.NoError(t, fs.MkdirAll(filepath.Join(root, "sub"), 0o750))
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
writeTestFile(t, fs, filepath.Join(root, "listing.mf"), "an ordinary file")
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
// A new file gives freshen something to write.
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
opts = testOpts([]string{
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
assert.ElementsMatch(t, []string{testFileTxt, "listing.mf", "added.txt"},
manifestPaths(t, fs, manifestPath))
}
// TestFreshenLeavesLeftoverTempFileOutOfListing freshens a manifest where
// an interrupted run left its temp file, index.mf.tmp, beside it: the
// leftover is not listed.
func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
t.Parallel()
root := t.TempDir()
manifestPath := filepath.Join(root, "index.mf")
fs := afero.NewOsFs()
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
writeTestFile(t, fs, manifestPath+".tmp", "part of a manifest")
// A new file gives freshen something to write.
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
opts = testOpts([]string{
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
assert.ElementsMatch(t, []string{testFileTxt, "added.txt"},
manifestPaths(t, fs, manifestPath))
}
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry // TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
// with respect to MFFilePath.Mtime, which is a message pointer with // with respect to MFFilePath.Mtime, which is a message pointer with
// proto3 field presence and may legitimately be absent. // proto3 field presence and may legitimately be absent.
+1
View File
@@ -94,6 +94,7 @@ func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
FollowSymLinks: ctx.Bool("follow-symlinks"), FollowSymLinks: ctx.Bool("follow-symlinks"),
IncludeTimestamps: ctx.Bool("include-timestamps"), IncludeTimestamps: ctx.Bool("include-timestamps"),
Fs: mfa.Fs, Fs: mfa.Fs,
OutputPath: ctx.String("output"),
} }
// Set seed for deterministic UUID if provided // Set seed for deterministic UUID if provided
+1 -1
View File
@@ -65,7 +65,7 @@ func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
} }
// resolveManifestArg resolves the manifest path from CLI arguments. // resolveManifestArg resolves the manifest path from CLI arguments.
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf. // HTTP(S) URLs are returned as-is. Directories are searched for index.mf.
// If no argument is given, the current directory is searched. // If no argument is given, the current directory is searched.
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) { func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
if ctx.Args().Len() > 0 { if ctx.Args().Len() > 0 {
+6 -1
View File
@@ -25,6 +25,11 @@ const (
flagProgress = "progress" flagProgress = "progress"
manifestArgsUsage = "[manifest file]" manifestArgsUsage = "[manifest file]"
// defaultManifestName is the filename gen writes by default, the one
// looked for when a command is given a directory, and the one fetch
// appends to a directory URL.
defaultManifestName = "index.mf"
) )
// errUnknownCommand indicates an unrecognized command argument. // errUnknownCommand indicates an unrecognized command argument.
@@ -156,7 +161,7 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
}, },
&cli.StringFlag{ &cli.StringFlag{
Name: "output", Name: "output",
Value: "./.index.mf", Value: defaultManifestName,
Aliases: []string{"o"}, Aliases: []string{"o"},
Usage: "Specify output filename", Usage: "Specify output filename",
}, },
+25
View File
@@ -5,10 +5,12 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"path/filepath"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -397,6 +399,29 @@ func TestNewManifestFromReaderTruncated(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestNewManifestFromFileRequiresPath(t *testing.T) {
t.Parallel()
_, err := NewManifestFromFile(nil)
require.ErrorIs(t, err, errManifestPathEmpty)
_, err = NewManifestFromFile(&ManifestFromFileOptions{Fs: afero.NewMemMapFs()})
require.ErrorIs(t, err, errManifestPathEmpty)
}
func TestNewManifestFromFileNilFsUsesOsFs(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "index.mf")
createTestManifest(t, afero.NewOsFs(), path, map[string][]byte{
testFileName: []byte("hello"),
})
m, err := NewManifestFromFile(&ManifestFromFileOptions{Path: path})
require.NoError(t, err)
assert.Len(t, m.Files(), 1)
}
func TestManifestString(t *testing.T) { func TestManifestString(t *testing.T) {
t.Parallel() t.Parallel()
+34 -8
View File
@@ -14,7 +14,11 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
) )
var errNoSigningPubKey = errors.New("manifest has no signing public key") var (
errNoSigningPubKey = errors.New("manifest has no signing public key")
errManifestPathEmpty = errors.New("manifest path cannot be empty")
errBasePathEmpty = errors.New("base path cannot be empty")
)
// Result represents the outcome of checking a single file. // Result represents the outcome of checking a single file.
type Result struct { type Result struct {
@@ -82,20 +86,42 @@ type Checker struct {
signingPubKey []byte signingPubKey []byte
} }
// NewChecker creates a new Checker for the given manifest, base path, and filesystem. // CheckerOptions configures a Checker.
// The basePath is the directory relative to which manifest paths are resolved. type CheckerOptions struct {
// If fs is nil, the real filesystem (OsFs) is used. // ManifestPath is the manifest file to check against (required).
func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, error) { ManifestPath string
// BasePath is the directory relative to which manifest paths are
// resolved (required).
BasePath string
// Fs is the filesystem to use, defaults to OsFs if nil.
Fs afero.Fs
}
// NewChecker creates a new Checker with the given options. It returns an
// error if opts is nil or either path is empty.
func NewChecker(opts *CheckerOptions) (*Checker, error) {
if opts == nil || opts.ManifestPath == "" {
return nil, errManifestPathEmpty
}
if opts.BasePath == "" {
return nil, errBasePathEmpty
}
fs := opts.Fs
if fs == nil { if fs == nil {
fs = afero.NewOsFs() fs = afero.NewOsFs()
} }
m, err := NewManifestFromFile(fs, manifestPath) m, err := NewManifestFromFile(&ManifestFromFileOptions{
Path: opts.ManifestPath,
Fs: fs,
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
abs, err := filepath.Abs(basePath) abs, err := filepath.Abs(opts.BasePath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -108,7 +134,7 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
} }
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles // Compute manifest's relative path from basePath for exclusion in FindExtraFiles
absManifest, err := filepath.Abs(manifestPath) absManifest, err := filepath.Abs(opts.ManifestPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+196 -50
View File
@@ -5,6 +5,8 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"os"
"path/filepath"
"testing" "testing"
"time" "time"
@@ -14,9 +16,13 @@ import (
) )
const ( const (
testFile1 = "file1.txt" testFile1 = "file1.txt"
testFile2 = "file2.txt" testFile2 = "file2.txt"
testExistsFile = "exists.txt" testExistsFile = "exists.txt"
testManifestPath = "/manifest.mf"
testDataDir = "/data"
// testDataManifestPath is a manifest kept inside the checked tree.
testDataManifestPath = testDataDir + "/index.mf"
) )
func TestStatusString(t *testing.T) { func TestStatusString(t *testing.T) {
@@ -66,15 +72,13 @@ func createTestManifest(
} }
// createFilesOnDisk creates the given files on the filesystem under // createFilesOnDisk creates the given files on the filesystem under
// /data. // testDataDir.
func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) { func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) {
t.Helper() t.Helper()
basePath := "/data"
for path, content := range files { for path, content := range files {
fullPath := basePath + "/" + path fullPath := testDataDir + "/" + path
require.NoError(t, fs.MkdirAll(basePath, 0o755)) require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644)) require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
} }
} }
@@ -90,9 +94,13 @@ func TestNewChecker(t *testing.T) {
testFile1: []byte("hello"), testFile1: []byte("hello"),
testFile2: []byte("world"), testFile2: []byte("world"),
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
chk, err := NewChecker("/manifest.mf", "/", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: "/",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.NotNil(t, chk) assert.NotNil(t, chk)
assert.Equal(t, FileCount(2), chk.FileCount()) assert.Equal(t, FileCount(2), chk.FileCount())
@@ -102,7 +110,11 @@ func TestNewChecker(t *testing.T) {
t.Parallel() t.Parallel()
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
_, err := NewChecker("/nonexistent.mf", "/", fs) _, err := NewChecker(&CheckerOptions{
ManifestPath: "/nonexistent.mf",
BasePath: "/",
Fs: fs,
})
assert.Error(t, err) assert.Error(t, err)
}) })
@@ -111,11 +123,73 @@ func TestNewChecker(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644)) require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
_, err := NewChecker("/bad.mf", "/", fs) _, err := NewChecker(&CheckerOptions{
ManifestPath: "/bad.mf",
BasePath: "/",
Fs: fs,
})
assert.Error(t, err) assert.Error(t, err)
}) })
} }
func TestNewCheckerRequiredPaths(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
opts *CheckerOptions
want string
is error
}{
{
name: "nil options",
opts: nil,
want: "manifest path cannot be empty",
is: errManifestPathEmpty,
},
{
name: "empty manifest path",
opts: &CheckerOptions{BasePath: testDataDir},
want: "manifest path cannot be empty",
is: errManifestPathEmpty,
},
{
name: "empty base path",
opts: &CheckerOptions{ManifestPath: testManifestPath},
want: "base path cannot be empty",
is: errBasePathEmpty,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
chk, err := NewChecker(tc.opts)
require.ErrorIs(t, err, tc.is)
require.EqualError(t, err, tc.want)
assert.Nil(t, chk)
})
}
}
func TestNewCheckerNilFsUsesOsFs(t *testing.T) {
t.Parallel()
dir := t.TempDir()
manifestPath := filepath.Join(dir, "index.mf")
content := []byte("hello")
createTestManifest(t, afero.NewOsFs(), manifestPath, map[string][]byte{
testFile1: content,
})
require.NoError(t, os.WriteFile(filepath.Join(dir, testFile1), content, 0o600))
chk, err := NewChecker(&CheckerOptions{ManifestPath: manifestPath, BasePath: dir})
require.NoError(t, err)
results := make(chan Result, 1)
require.NoError(t, chk.Check(context.Background(), results, nil))
assert.Equal(t, StatusOK, (<-results).Status)
}
func TestCheckerFileCountAndTotalBytes(t *testing.T) { func TestCheckerFileCountAndTotalBytes(t *testing.T) {
t.Parallel() t.Parallel()
@@ -125,9 +199,13 @@ func TestCheckerFileCountAndTotalBytes(t *testing.T) {
"medium.txt": []byte("hello world"), "medium.txt": []byte("hello world"),
"large.txt": bytes.Repeat([]byte("x"), 1000), "large.txt": bytes.Repeat([]byte("x"), 1000),
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
chk, err := NewChecker("/manifest.mf", "/", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: "/",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, FileCount(3), chk.FileCount()) assert.Equal(t, FileCount(3), chk.FileCount())
@@ -142,10 +220,14 @@ func TestCheckAllFilesOK(t *testing.T) {
testFile1: []byte("content one"), testFile1: []byte("content one"),
testFile2: []byte("content two"), testFile2: []byte("content two"),
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -172,13 +254,17 @@ func TestCheckMissingFile(t *testing.T) {
testExistsFile: []byte("I exist"), testExistsFile: []byte("I exist"),
"missing.txt": []byte("I don't exist on disk"), "missing.txt": []byte("I don't exist on disk"),
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
// Only create one file // Only create one file
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
testExistsFile: []byte("I exist"), testExistsFile: []byte("I exist"),
}) })
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -211,13 +297,17 @@ func TestCheckSizeMismatch(t *testing.T) {
files := map[string][]byte{ files := map[string][]byte{
testFileName: []byte("original content"), testFileName: []byte("original content"),
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
// Create file with different size // Create file with different size
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
testFileName: []byte("short"), testFileName: []byte("short"),
}) })
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -237,7 +327,7 @@ func TestCheckHashMismatch(t *testing.T) {
files := map[string][]byte{ files := map[string][]byte{
testFileName: originalContent, testFileName: originalContent,
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
// Create file with same size but different content // Create file with same size but different content
differentContent := []byte("different contnt") // same length (16 bytes) but different differentContent := []byte("different contnt") // same length (16 bytes) but different
require.Len(t, differentContent, len(originalContent), "test requires same length") require.Len(t, differentContent, len(originalContent), "test requires same length")
@@ -245,7 +335,11 @@ func TestCheckHashMismatch(t *testing.T) {
testFileName: differentContent, testFileName: differentContent,
}) })
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -265,10 +359,14 @@ func TestCheckWithProgress(t *testing.T) {
testFile1: bytes.Repeat([]byte("a"), 100), testFile1: bytes.Repeat([]byte("a"), 100),
testFile2: bytes.Repeat([]byte("b"), 200), testFile2: bytes.Repeat([]byte("b"), 200),
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -305,10 +403,14 @@ func TestCheckContextCancellation(t *testing.T) {
files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000) files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000)
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@@ -327,7 +429,7 @@ func TestFindExtraFiles(t *testing.T) {
manifestFiles := map[string][]byte{ manifestFiles := map[string][]byte{
testFile1: []byte("in manifest"), testFile1: []byte("in manifest"),
} }
createTestManifest(t, fs, "/manifest.mf", manifestFiles) createTestManifest(t, fs, testManifestPath, manifestFiles)
// Disk has file1 and file2 // Disk has file1 and file2
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
@@ -335,7 +437,11 @@ func TestFindExtraFiles(t *testing.T) {
testFile2: []byte("extra file"), testFile2: []byte("extra file"),
}) })
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -360,7 +466,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
manifestFiles := map[string][]byte{ manifestFiles := map[string][]byte{
testFile1: []byte("in manifest"), testFile1: []byte("in manifest"),
} }
createTestManifest(t, fs, "/data/.index.mf", manifestFiles) createTestManifest(t, fs, testDataManifestPath, manifestFiles)
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
testFile1: []byte("in manifest"), testFile1: []byte("in manifest"),
}) })
@@ -368,10 +474,14 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644)) require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644)) require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
// Create a real extra file // Create a real extra file
require.NoError(t, fs.MkdirAll("/data", 0o755)) require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644)) require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
chk, err := NewChecker("/data/.index.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testDataManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -383,7 +493,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
extras = append(extras, r) extras = append(extras, r)
} }
// Should only report extra.txt, not .hidden, .config/settings, or .index.mf // Should only report extra.txt, not .hidden, .config/settings, or index.mf
for _, e := range extras { for _, e := range extras {
t.Logf("extra: %s", e.Path) t.Logf("extra: %s", e.Path)
} }
@@ -400,10 +510,14 @@ func TestFindExtraFilesContextCancellation(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
files := map[string][]byte{testFileName: []byte("data")} files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@@ -419,10 +533,14 @@ func TestCheckNilChannels(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
files := map[string][]byte{testFileName: []byte("data")} files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
// Should not panic with nil channels // Should not panic with nil channels
@@ -435,10 +553,14 @@ func TestFindExtraFilesNilChannel(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
files := map[string][]byte{testFileName: []byte("data")} files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
// Should not panic with nil channel // Should not panic with nil channel
@@ -455,7 +577,7 @@ func TestCheckSubdirectories(t *testing.T) {
"dir1/dir2/file2.txt": []byte("content2"), "dir1/dir2/file2.txt": []byte("content2"),
"dir1/dir2/dir3/deep.txt": []byte("deep content"), "dir1/dir2/dir3/deep.txt": []byte("deep content"),
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
// Create files with full directory structure // Create files with full directory structure
for path, content := range files { for path, content := range files {
@@ -465,7 +587,11 @@ func TestCheckSubdirectories(t *testing.T) {
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644)) require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
} }
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -493,13 +619,17 @@ func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
testExistsFile: []byte("here"), testExistsFile: []byte("here"),
"missing.txt": []byte("not on disk"), "missing.txt": []byte("not on disk"),
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
// Only create one file on disk // Only create one file on disk
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
testExistsFile: []byte("here"), testExistsFile: []byte("here"),
}) })
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -528,7 +658,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
files := map[string][]byte{ files := map[string][]byte{
testFile1: []byte("in manifest"), testFile1: []byte("in manifest"),
} }
createTestManifest(t, fs, "/data/.index.mf", files) createTestManifest(t, fs, testDataManifestPath, files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
// Add dotfiles and manifest file on disk // Add dotfiles and manifest file on disk
@@ -537,7 +667,11 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
require.NoError(t, require.NoError(t,
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644)) afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
chk, err := NewChecker("/data/.index.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testDataManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -563,10 +697,14 @@ func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
files := map[string][]byte{ files := map[string][]byte{
testFile1: []byte("content"), testFile1: []byte("content"),
} }
createTestManifest(t, fs, "/data/index.mf", files) createTestManifest(t, fs, testDataManifestPath, files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/data/index.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testDataManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -587,9 +725,13 @@ func TestCheckEmptyManifest(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
// Create manifest with no files // Create manifest with no files
createTestManifest(t, fs, "/manifest.mf", map[string][]byte{}) createTestManifest(t, fs, testManifestPath, map[string][]byte{})
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, FileCount(0), chk.FileCount()) assert.Equal(t, FileCount(0), chk.FileCount())
@@ -621,10 +763,14 @@ func TestCheckProgressRateLimited(t *testing.T) {
files[name] = []byte("content") files[name] = []byte("content")
} }
createTestManifest(t, fs, "/manifest.mf", files) createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 200) results := make(chan Result, 200)
+17 -4
View File
@@ -352,16 +352,29 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
return m, nil return m, nil
} }
// NewManifestFromFile reads a manifest from a file path using the given filesystem. // ManifestFromFileOptions configures NewManifestFromFile.
// If fs is nil, the real filesystem (OsFs) is used. type ManifestFromFileOptions struct {
// Path is the manifest file to read (required).
Path string
// Fs is the filesystem to use, defaults to OsFs if nil.
Fs afero.Fs
}
// NewManifestFromFile reads a manifest from a file. It returns an error if
// opts is nil or its path is empty.
// //
//nolint:revive // unexported-return: exporting manifest is owner question 13 //nolint:revive // unexported-return: exporting manifest is owner question 13
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) { func NewManifestFromFile(opts *ManifestFromFileOptions) (*manifest, error) {
if opts == nil || opts.Path == "" {
return nil, errManifestPathEmpty
}
fs := opts.Fs
if fs == nil { if fs == nil {
fs = afero.NewOsFs() fs = afero.NewOsFs()
} }
f, err := fs.Open(path) f, err := fs.Open(opts.Path)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+35 -1
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"io" "io"
"io/fs" "io/fs"
"os"
"path" "path"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -57,6 +58,15 @@ type ScannerOptions struct {
SigningOptions *SigningOptions SigningOptions *SigningOptions
// Seed, if set, derives a deterministic UUID from this seed. // Seed, if set, derives a deterministic UUID from this seed.
Seed string Seed string
// OutputPath, if set, is the file the manifest will be written to.
// Enumeration leaves out that file and OutputPath + ".tmp", the temp
// file mfer gen writes first, if they exist when the scanner is
// created, however their paths are spelled (through a symlink, a hard
// link, or in different letter case), so neither a manifest being
// replaced nor a temp file left by an interrupted run is ever listed.
// Files are matched with os.SameFile, which only recognizes files on
// the operating system's filesystem.
OutputPath string
} }
// FileEntry represents a file that has been enumerated. // FileEntry represents a file that has been enumerated.
@@ -75,6 +85,8 @@ type Scanner struct {
totalBytes FileSize // cached sum of all file sizes totalBytes FileSize // cached sum of all file sizes
options *ScannerOptions options *ScannerOptions
fs afero.Fs fs afero.Fs
outputInfo fs.FileInfo // the existing output file; nil when there is none
tmpInfo fs.FileInfo // its leftover temp file; nil when there is none
} }
// NewScanner creates a new Scanner with default options. // NewScanner creates a new Scanner with default options.
@@ -93,11 +105,27 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
fs = afero.NewOsFs() fs = afero.NewOsFs()
} }
return &Scanner{ s := &Scanner{
files: make([]*FileEntry, 0), files: make([]*FileEntry, 0),
options: opts, options: opts,
fs: fs, fs: fs,
} }
if opts.OutputPath != "" {
// A file that cannot be stat'd, normally because it does not
// exist, needs no leaving out.
info, err := s.fs.Stat(opts.OutputPath)
if err == nil {
s.outputInfo = info
}
info, err = s.fs.Stat(opts.OutputPath + ".tmp")
if err == nil {
s.tmpInfo = info
}
}
return s
} }
// EnumerateFile adds a single file to the scanner, calling stat() to get metadata. // EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
@@ -435,6 +463,12 @@ func (s *Scanner) enumerateFileWithInfo(
info = realInfo info = realInfo
} }
// Neither the manifest being written nor its temp file is one of the
// files it lists. os.SameFile never matches a nil FileInfo.
if os.SameFile(info, s.outputInfo) || os.SameFile(info, s.tmpInfo) {
return nil
}
entry := &FileEntry{ entry := &FileEntry{
Path: RelFilePath(cleanPath), Path: RelFilePath(cleanPath),
AbsPath: AbsFilePath(absPath), AbsPath: AbsFilePath(absPath),