Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0019f9afbf |
+1
-1
@@ -8,7 +8,7 @@ vendor.tzst
|
|||||||
modcache.tzst
|
modcache.tzst
|
||||||
|
|
||||||
# Generated manifest files
|
# Generated manifest files
|
||||||
/index.mf
|
.index.mf
|
||||||
|
|
||||||
# Secrets
|
# Secrets
|
||||||
.env
|
.env
|
||||||
|
|||||||
@@ -36,12 +36,12 @@ Generate a manifest for a directory tree, verify it later, and fetch a published
|
|||||||
tree by URL:
|
tree by URL:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
# Write index.mf, a manifest of the files under the current directory.
|
# Write .index.mf, a manifest of the files under the current directory.
|
||||||
bin/mfer gen .
|
bin/mfer gen .
|
||||||
|
|
||||||
# Verify the files on disk against the manifest. Exits nonzero if any file
|
# Verify the files on disk against the manifest. Exits nonzero if any file
|
||||||
# is missing or corrupted.
|
# is missing or corrupted.
|
||||||
bin/mfer check index.mf
|
bin/mfer check .index.mf
|
||||||
|
|
||||||
# Download and cryptographically verify a tree published over HTTP: mfer
|
# Download and cryptographically verify a tree published over HTTP: mfer
|
||||||
# fetches <url>/index.mf, then downloads every file it lists.
|
# fetches <url>/index.mf, then downloads every file it lists.
|
||||||
|
|||||||
+10
-7
@@ -75,22 +75,25 @@ func safeRateUint64(rate float64) uint64 {
|
|||||||
return uint64(rate)
|
return uint64(rate)
|
||||||
}
|
}
|
||||||
|
|
||||||
// findManifest returns the path of the manifest with the default name in
|
// findManifest looks for a manifest file in the given directory.
|
||||||
// dir, or an error if there is none.
|
// It checks for index.mf and .index.mf, returning the first one found.
|
||||||
func findManifest(fs afero.Fs, dir string) (string, error) {
|
func findManifest(fs afero.Fs, dir string) (string, error) {
|
||||||
path := filepath.Join(dir, defaultManifestName)
|
candidates := []string{"index.mf", ".index.mf"}
|
||||||
|
for _, name := range candidates {
|
||||||
|
path := filepath.Join(dir, name)
|
||||||
|
|
||||||
exists, err := afero.Exists(fs, path)
|
exists, err := afero.Exists(fs, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
if !exists {
|
if exists {
|
||||||
return "", fmt.Errorf("%w in %s (looked for %s)",
|
return path, nil
|
||||||
errNoManifestFound, dir, defaultManifestName)
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return path, nil
|
return "", fmt.Errorf(
|
||||||
|
"%w in %s (looked for index.mf and .index.mf)", errNoManifestFound, dir)
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchManifestToTemp downloads a manifest URL to a temporary file and
|
// fetchManifestToTemp downloads a manifest URL to a temporary file and
|
||||||
|
|||||||
+5
-101
@@ -8,7 +8,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -28,6 +27,7 @@ const (
|
|||||||
testFile1 = "/testdir/file1.txt"
|
testFile1 = "/testdir/file1.txt"
|
||||||
testMF = "/testdir/test.mf"
|
testMF = "/testdir/test.mf"
|
||||||
testOutput = "/output.mf"
|
testOutput = "/output.mf"
|
||||||
|
testOutputTmp = "/output.mf.tmp"
|
||||||
testManifest = "/manifest.mf"
|
testManifest = "/manifest.mf"
|
||||||
testFlagBase = "--base"
|
testFlagBase = "--base"
|
||||||
testFlagNoExtra = "--no-extra-files"
|
testFlagNoExtra = "--no-extra-files"
|
||||||
@@ -682,7 +682,7 @@ func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
|
|||||||
assert.True(t, exists, "output file should exist")
|
assert.True(t, exists, "output file should exist")
|
||||||
|
|
||||||
// Verify temp file does NOT exist
|
// Verify temp file does NOT exist
|
||||||
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.False(t, tmpExists,
|
assert.False(t, tmpExists,
|
||||||
"temp file should not exist after successful generation")
|
"temp file should not exist after successful generation")
|
||||||
@@ -714,7 +714,7 @@ func TestGenerateAtomicWriteOverwriteWithForce(t *testing.T) {
|
|||||||
"manifest should be overwritten")
|
"manifest should be overwritten")
|
||||||
|
|
||||||
// Verify temp file does NOT exist
|
// Verify temp file does NOT exist
|
||||||
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.False(t, tmpExists,
|
assert.False(t, tmpExists,
|
||||||
"temp file should not exist after successful generation")
|
"temp file should not exist after successful generation")
|
||||||
@@ -743,102 +743,6 @@ func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
|
|||||||
assert.Equal(t, "existing", string(content), "original file should be preserved")
|
assert.Equal(t, "existing", string(content), "original file should be preserved")
|
||||||
}
|
}
|
||||||
|
|
||||||
// manifestPaths returns the file paths listed by the manifest at path.
|
|
||||||
func manifestPaths(t *testing.T, fs afero.Fs, path string) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
|
||||||
Path: path,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
paths := make([]string, 0, len(manifest.Files()))
|
|
||||||
for _, f := range manifest.Files() {
|
|
||||||
paths = append(paths, f.GetPath())
|
|
||||||
}
|
|
||||||
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGenerateLeavesOutputOutOfListing overwrites an output file inside the
|
|
||||||
// scanned tree with --force: the old file is not listed, even when the tree
|
|
||||||
// is named through a symlink, while a file named index.mf in a subdirectory
|
|
||||||
// still is. The output file is recognized by file identity, which needs
|
|
||||||
// the real filesystem.
|
|
||||||
func TestGenerateLeavesOutputOutOfListing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Paths are relative to a temp dir holding data/tree and link, a
|
|
||||||
// symlink to data.
|
|
||||||
for name, tc := range map[string]struct{ input, output string }{
|
|
||||||
"default name": {"data/tree", "data/tree/index.mf"},
|
|
||||||
"other name in a subdirectory": {"data/tree", "data/tree/sub/listing.mf"},
|
|
||||||
"tree named through a symlink": {"link/tree", "data/tree/index.mf"},
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
root := t.TempDir()
|
|
||||||
tree := filepath.Join(root, "data", "tree")
|
|
||||||
output := filepath.Join(root, tc.output)
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
require.NoError(t, fs.MkdirAll(filepath.Join(tree, "sub"), 0o750))
|
|
||||||
require.NoError(t,
|
|
||||||
os.Symlink(filepath.Join(root, "data"), filepath.Join(root, "link")))
|
|
||||||
writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
|
|
||||||
writeTestFile(t, fs, filepath.Join(tree, "sub", "index.mf"), "an ordinary file")
|
|
||||||
writeTestFile(t, fs, output, "previous manifest")
|
|
||||||
|
|
||||||
opts := testOpts([]string{
|
|
||||||
testApp, cmdGenerate, "-q", "--force", "-o", output, filepath.Join(root, tc.input),
|
|
||||||
}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, []string{testFileTxt, "sub/index.mf"},
|
|
||||||
manifestPaths(t, fs, output))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGenerateDefaultOutputLeftOutOfListing runs gen with --force and no
|
|
||||||
// other arguments, so it scans the current directory and writes the
|
|
||||||
// relative path index.mf: the index.mf already there is not listed.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // changes the process-global working directory
|
|
||||||
func TestGenerateDefaultOutputLeftOutOfListing(t *testing.T) {
|
|
||||||
chdirTemp(t)
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
writeTestFile(t, fs, testFileTxt, "hello")
|
|
||||||
writeTestFile(t, fs, "index.mf", "previous manifest")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "--force"}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, "index.mf"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGenerateLeavesLeftoverTempFileOutOfListing runs gen where an
|
|
||||||
// interrupted run left its temp file beside the output: the leftover is
|
|
||||||
// not listed, and gen does not fail when it overwrites it.
|
|
||||||
func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
root := t.TempDir()
|
|
||||||
output := filepath.Join(root, "index.mf")
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
|
||||||
writeTestFile(t, fs, manifestTempPath(output), "part of a manifest")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", output, root}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -861,7 +765,7 @@ func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
|||||||
exists, _ := afero.Exists(fs, testOutput)
|
exists, _ := afero.Exists(fs, testOutput)
|
||||||
assert.True(t, exists, "output file should exist")
|
assert.True(t, exists, "output file should exist")
|
||||||
|
|
||||||
tmpExists, _ := afero.Exists(fs, manifestTempPath(testOutput))
|
tmpExists, _ := afero.Exists(fs, testOutputTmp)
|
||||||
assert.False(t, tmpExists, "temp file should be cleaned up")
|
assert.False(t, tmpExists, "temp file should be cleaned up")
|
||||||
|
|
||||||
// Verify manifest is valid (not empty)
|
// Verify manifest is valid (not empty)
|
||||||
@@ -927,7 +831,7 @@ func TestGenerateAtomicWriteCleansUpOnError(t *testing.T) {
|
|||||||
"output file should not exist after failed generation (atomic write)")
|
"output file should not exist after failed generation (atomic write)")
|
||||||
|
|
||||||
// Temp file should also not exist
|
// Temp file should also not exist
|
||||||
tmpExists, _ := afero.Exists(baseFs, manifestTempPath(testOutput))
|
tmpExists, _ := afero.Exists(baseFs, testOutputTmp)
|
||||||
assert.False(t, tmpExists,
|
assert.False(t, tmpExists,
|
||||||
"temp file should be cleaned up after failed generation")
|
"temp file should be cleaned up after failed generation")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ func TestNoManifestFoundMessage(t *testing.T) {
|
|||||||
_, err := findManifest(afero.NewMemMapFs(), "/tmp/x")
|
_, err := findManifest(afero.NewMemMapFs(), "/tmp/x")
|
||||||
require.ErrorIs(t, err, errNoManifestFound)
|
require.ErrorIs(t, err, errNoManifestFound)
|
||||||
assert.EqualError(t, err,
|
assert.EqualError(t, err,
|
||||||
"no manifest found in /tmp/x (looked for index.mf)")
|
"no manifest found in /tmp/x (looked for index.mf and .index.mf)")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestVerifyRequiredSignerMessages(t *testing.T) {
|
func TestVerifyRequiredSignerMessages(t *testing.T) {
|
||||||
|
|||||||
@@ -313,7 +313,7 @@ func checkNoSymlinks(p string) error {
|
|||||||
|
|
||||||
// resolveManifestURL takes a URL and returns the manifest URL.
|
// resolveManifestURL takes a URL and returns the manifest URL.
|
||||||
// If the URL already ends with .mf, it's returned as-is.
|
// If the URL already ends with .mf, it's returned as-is.
|
||||||
// Otherwise, the default manifest name is appended.
|
// Otherwise, index.mf is appended.
|
||||||
func resolveManifestURL(inputURL string) (string, error) {
|
func resolveManifestURL(inputURL string) (string, error) {
|
||||||
parsed, err := url.Parse(inputURL)
|
parsed, err := url.Parse(inputURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -330,7 +330,8 @@ func resolveManifestURL(inputURL string) (string, error) {
|
|||||||
parsed.Path += "/"
|
parsed.Path += "/"
|
||||||
}
|
}
|
||||||
|
|
||||||
parsed.Path += defaultManifestName
|
// Append index.mf
|
||||||
|
parsed.Path += "index.mf"
|
||||||
|
|
||||||
return parsed.String(), nil
|
return parsed.String(), nil
|
||||||
}
|
}
|
||||||
|
|||||||
+8
-22
@@ -7,7 +7,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -57,7 +56,7 @@ type freshenEntry struct {
|
|||||||
type freshenScanner struct {
|
type freshenScanner struct {
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
absBase string
|
absBase string
|
||||||
excluded []fs.FileInfo // files left out of the listing
|
manifestBase string
|
||||||
includeDotfiles bool
|
includeDotfiles bool
|
||||||
followSymlinks bool
|
followSymlinks bool
|
||||||
showProgress bool
|
showProgress bool
|
||||||
@@ -157,6 +156,11 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
|||||||
"freshen: failed to compute relative path for %s: %w", path, err)
|
"freshen: failed to compute relative path for %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Skip the manifest file itself
|
||||||
|
if relPath == s.manifestBase || relPath == "."+s.manifestBase {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Handle dotfiles
|
// Handle dotfiles
|
||||||
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
|
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
@@ -181,12 +185,6 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
|||||||
info = realInfo
|
info = realInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, excluded := range s.excluded {
|
|
||||||
if os.SameFile(info, excluded) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
s.scanCount++
|
s.scanCount++
|
||||||
|
|
||||||
// Check against existing manifest
|
// Check against existing manifest
|
||||||
@@ -309,7 +307,7 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
|||||||
func writeFreshenedManifest(
|
func writeFreshenedManifest(
|
||||||
ctx context.Context, afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
ctx context.Context, afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
||||||
) error {
|
) error {
|
||||||
tmpPath := manifestTempPath(manifestPath)
|
tmpPath := manifestPath + ".tmp"
|
||||||
|
|
||||||
outFile, err := afs.Create(tmpPath)
|
outFile, err := afs.Create(tmpPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -366,22 +364,10 @@ func (mfa *CLIApp) freshenScan(
|
|||||||
startScan := time.Now()
|
startScan := time.Now()
|
||||||
showProgress := ctx.Bool("progress")
|
showProgress := ctx.Bool("progress")
|
||||||
|
|
||||||
// Leave out the manifest and a temp file left by an interrupted run,
|
|
||||||
// as gen does. A path that cannot be stat'd, normally because no file
|
|
||||||
// is there, needs no leaving out.
|
|
||||||
var excluded []fs.FileInfo
|
|
||||||
|
|
||||||
for _, p := range []string{manifestPath, manifestTempPath(manifestPath)} {
|
|
||||||
info, err := mfa.Fs.Stat(p)
|
|
||||||
if err == nil {
|
|
||||||
excluded = append(excluded, info)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
scanner := &freshenScanner{
|
scanner := &freshenScanner{
|
||||||
fs: mfa.Fs,
|
fs: mfa.Fs,
|
||||||
absBase: absBase,
|
absBase: absBase,
|
||||||
excluded: excluded,
|
manifestBase: filepath.Base(manifestPath),
|
||||||
includeDotfiles: ctx.Bool("include-dotfiles"),
|
includeDotfiles: ctx.Bool("include-dotfiles"),
|
||||||
followSymlinks: ctx.Bool("follow-symlinks"),
|
followSymlinks: ctx.Bool("follow-symlinks"),
|
||||||
showProgress: showProgress,
|
showProgress: showProgress,
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -32,7 +31,7 @@ func (s stubFileInfo) IsDir() bool { return false }
|
|||||||
func (s stubFileInfo) Sys() any { return nil }
|
func (s stubFileInfo) Sys() any { return nil }
|
||||||
|
|
||||||
// setupFreshenDir populates /testdir with two files, scans it, and
|
// setupFreshenDir populates /testdir with two files, scans it, and
|
||||||
// writes the resulting manifest to /testdir/index.mf.
|
// writes the resulting manifest to /testdir/.index.mf.
|
||||||
func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -51,7 +50,7 @@ func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
|||||||
|
|
||||||
// Write manifest to filesystem
|
// Write manifest to filesystem
|
||||||
require.NoError(t,
|
require.NoError(t,
|
||||||
afero.WriteFile(fs, "/testdir/index.mf", manifestBuf.Bytes(), 0o644))
|
afero.WriteFile(fs, "/testdir/.index.mf", manifestBuf.Bytes(), 0o644))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFreshenUnchanged(t *testing.T) {
|
func TestFreshenUnchanged(t *testing.T) {
|
||||||
@@ -62,7 +61,7 @@ func TestFreshenUnchanged(t *testing.T) {
|
|||||||
|
|
||||||
// Parse manifest to verify
|
// Parse manifest to verify
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||||
Path: "/testdir/index.mf",
|
Path: "/testdir/.index.mf",
|
||||||
Fs: fs,
|
Fs: fs,
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -77,7 +76,7 @@ func TestFreshenWithChanges(t *testing.T) {
|
|||||||
|
|
||||||
// Verify initial manifest has 2 files
|
// Verify initial manifest has 2 files
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||||
Path: "/testdir/index.mf",
|
Path: "/testdir/.index.mf",
|
||||||
Fs: fs,
|
Fs: fs,
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -104,64 +103,6 @@ func TestFreshenWithChanges(t *testing.T) {
|
|||||||
assert.Equal(t, "modified content2", string(content))
|
assert.Equal(t, "modified content2", string(content))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a
|
|
||||||
// subdirectory of the tree it lists: the manifest is not listed, while an
|
|
||||||
// ordinary file of the same name at the top of the tree is. The manifest
|
|
||||||
// is recognized by file identity, which needs the real filesystem.
|
|
||||||
func TestFreshenLeavesManifestOutOfListing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
root := t.TempDir()
|
|
||||||
manifestPath := filepath.Join(root, "sub", "listing.mf")
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
require.NoError(t, fs.MkdirAll(filepath.Join(root, "sub"), 0o750))
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, "listing.mf"), "an ordinary file")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
// A new file gives freshen something to write.
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
|
||||||
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
|
|
||||||
}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, []string{testFileTxt, "listing.mf", "added.txt"},
|
|
||||||
manifestPaths(t, fs, manifestPath))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFreshenLeavesLeftoverTempFileOutOfListing freshens a manifest where
|
|
||||||
// an interrupted run left its temp file beside it: the leftover is not
|
|
||||||
// listed.
|
|
||||||
func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
root := t.TempDir()
|
|
||||||
manifestPath := filepath.Join(root, "index.mf")
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
writeTestFile(t, fs, manifestTempPath(manifestPath), "part of a manifest")
|
|
||||||
// A new file gives freshen something to write.
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
|
||||||
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
|
|
||||||
}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, []string{testFileTxt, "added.txt"},
|
|
||||||
manifestPaths(t, fs, manifestPath))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
|
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
|
||||||
// with respect to MFFilePath.Mtime, which is a message pointer with
|
// with respect to MFFilePath.Mtime, which is a message pointer with
|
||||||
// proto3 field presence and may legitimately be absent.
|
// proto3 field presence and may legitimately be absent.
|
||||||
|
|||||||
+1
-5
@@ -89,15 +89,11 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
|||||||
|
|
||||||
// buildScannerOptions constructs scanner options from the CLI flags.
|
// buildScannerOptions constructs scanner options from the CLI flags.
|
||||||
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
||||||
output := ctx.String("output")
|
|
||||||
opts := &mfer.ScannerOptions{
|
opts := &mfer.ScannerOptions{
|
||||||
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
||||||
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
||||||
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
||||||
Fs: mfa.Fs,
|
Fs: mfa.Fs,
|
||||||
// Neither a manifest being replaced nor a temp file left by an
|
|
||||||
// interrupted run belongs in the new manifest.
|
|
||||||
ExcludePaths: []string{output, manifestTempPath(output)},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set seed for deterministic UUID if provided
|
// Set seed for deterministic UUID if provided
|
||||||
@@ -221,7 +217,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Create temp file for atomic write
|
// Create temp file for atomic write
|
||||||
tmpPath := manifestTempPath(outputPath)
|
tmpPath := outputPath + ".tmp"
|
||||||
|
|
||||||
outFile, err := mfa.Fs.Create(tmpPath)
|
outFile, err := mfa.Fs.Create(tmpPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// resolveManifestArg resolves the manifest path from CLI arguments.
|
// resolveManifestArg resolves the manifest path from CLI arguments.
|
||||||
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf.
|
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf.
|
||||||
// If no argument is given, the current directory is searched.
|
// If no argument is given, the current directory is searched.
|
||||||
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
|
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
|
||||||
if ctx.Args().Len() > 0 {
|
if ctx.Args().Len() > 0 {
|
||||||
|
|||||||
+1
-12
@@ -25,19 +25,8 @@ const (
|
|||||||
flagProgress = "progress"
|
flagProgress = "progress"
|
||||||
|
|
||||||
manifestArgsUsage = "[manifest file]"
|
manifestArgsUsage = "[manifest file]"
|
||||||
|
|
||||||
// defaultManifestName is the filename gen writes by default, the one
|
|
||||||
// looked for when a command is given a directory, and the one fetch
|
|
||||||
// appends to a directory URL.
|
|
||||||
defaultManifestName = "index.mf"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// manifestTempPath returns the temp file gen and freshen write a manifest
|
|
||||||
// to before renaming it to out.
|
|
||||||
func manifestTempPath(out string) string {
|
|
||||||
return out + ".tmp"
|
|
||||||
}
|
|
||||||
|
|
||||||
// errUnknownCommand indicates an unrecognized command argument.
|
// errUnknownCommand indicates an unrecognized command argument.
|
||||||
var errUnknownCommand = errors.New("unknown command")
|
var errUnknownCommand = errors.New("unknown command")
|
||||||
|
|
||||||
@@ -167,7 +156,7 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
|||||||
},
|
},
|
||||||
&cli.StringFlag{
|
&cli.StringFlag{
|
||||||
Name: "output",
|
Name: "output",
|
||||||
Value: defaultManifestName,
|
Value: "./.index.mf",
|
||||||
Aliases: []string{"o"},
|
Aliases: []string{"o"},
|
||||||
Usage: "Specify output filename",
|
Usage: "Specify output filename",
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -94,9 +94,6 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
|||||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
hash Multihash
|
hash Multihash
|
||||||
@@ -109,8 +106,6 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
|||||||
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
|
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
|
||||||
// A valid multihash, but its 20-byte SHA-1 digest is too short.
|
// A valid multihash, but its 20-byte SHA-1 digest is too short.
|
||||||
{"SHA-1 multihash", sha1Hash, errHashTooShort},
|
{"SHA-1 multihash", sha1Hash, errHashTooShort},
|
||||||
// A valid multihash whose 31-byte digest is one byte short.
|
|
||||||
{"31-byte digest", truncatedHash, errHashTooShort},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -21,8 +21,6 @@ const (
|
|||||||
testExistsFile = "exists.txt"
|
testExistsFile = "exists.txt"
|
||||||
testManifestPath = "/manifest.mf"
|
testManifestPath = "/manifest.mf"
|
||||||
testDataDir = "/data"
|
testDataDir = "/data"
|
||||||
// testDataManifestPath is a manifest kept inside the checked tree.
|
|
||||||
testDataManifestPath = testDataDir + "/index.mf"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestStatusString(t *testing.T) {
|
func TestStatusString(t *testing.T) {
|
||||||
@@ -466,7 +464,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
|||||||
manifestFiles := map[string][]byte{
|
manifestFiles := map[string][]byte{
|
||||||
testFile1: []byte("in manifest"),
|
testFile1: []byte("in manifest"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testDataManifestPath, manifestFiles)
|
createTestManifest(t, fs, "/data/.index.mf", manifestFiles)
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
createFilesOnDisk(t, fs, map[string][]byte{
|
||||||
testFile1: []byte("in manifest"),
|
testFile1: []byte("in manifest"),
|
||||||
})
|
})
|
||||||
@@ -478,7 +476,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
|||||||
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
|
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker(&CheckerOptions{
|
||||||
ManifestPath: testDataManifestPath,
|
ManifestPath: "/data/.index.mf",
|
||||||
BasePath: testDataDir,
|
BasePath: testDataDir,
|
||||||
Fs: fs,
|
Fs: fs,
|
||||||
})
|
})
|
||||||
@@ -493,7 +491,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
|||||||
extras = append(extras, r)
|
extras = append(extras, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Should only report extra.txt, not .hidden, .config/settings, or index.mf
|
// Should only report extra.txt, not .hidden, .config/settings, or .index.mf
|
||||||
for _, e := range extras {
|
for _, e := range extras {
|
||||||
t.Logf("extra: %s", e.Path)
|
t.Logf("extra: %s", e.Path)
|
||||||
}
|
}
|
||||||
@@ -658,7 +656,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
|
|||||||
files := map[string][]byte{
|
files := map[string][]byte{
|
||||||
testFile1: []byte("in manifest"),
|
testFile1: []byte("in manifest"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testDataManifestPath, files)
|
createTestManifest(t, fs, "/data/.index.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
// Add dotfiles and manifest file on disk
|
// Add dotfiles and manifest file on disk
|
||||||
@@ -668,7 +666,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
|
|||||||
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
|
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker(&CheckerOptions{
|
||||||
ManifestPath: testDataManifestPath,
|
ManifestPath: "/data/.index.mf",
|
||||||
BasePath: testDataDir,
|
BasePath: testDataDir,
|
||||||
Fs: fs,
|
Fs: fs,
|
||||||
})
|
})
|
||||||
@@ -697,11 +695,11 @@ func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
|
|||||||
files := map[string][]byte{
|
files := map[string][]byte{
|
||||||
testFile1: []byte("content"),
|
testFile1: []byte("content"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testDataManifestPath, files)
|
createTestManifest(t, fs, "/data/index.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker(&CheckerOptions{
|
||||||
ManifestPath: testDataManifestPath,
|
ManifestPath: "/data/index.mf",
|
||||||
BasePath: testDataDir,
|
BasePath: testDataDir,
|
||||||
Fs: fs,
|
Fs: fs,
|
||||||
})
|
})
|
||||||
|
|||||||
+1
-22
@@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -58,8 +57,6 @@ type ScannerOptions struct {
|
|||||||
SigningOptions *SigningOptions
|
SigningOptions *SigningOptions
|
||||||
// Seed, if set, derives a deterministic UUID from this seed.
|
// Seed, if set, derives a deterministic UUID from this seed.
|
||||||
Seed string
|
Seed string
|
||||||
// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
|
|
||||||
ExcludePaths []string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FileEntry represents a file that has been enumerated.
|
// FileEntry represents a file that has been enumerated.
|
||||||
@@ -78,7 +75,6 @@ type Scanner struct {
|
|||||||
totalBytes FileSize // cached sum of all file sizes
|
totalBytes FileSize // cached sum of all file sizes
|
||||||
options *ScannerOptions
|
options *ScannerOptions
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
excluded []fs.FileInfo // the files named in ExcludePaths that exist
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewScanner creates a new Scanner with default options.
|
// NewScanner creates a new Scanner with default options.
|
||||||
@@ -97,22 +93,11 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
|
|||||||
fs = afero.NewOsFs()
|
fs = afero.NewOsFs()
|
||||||
}
|
}
|
||||||
|
|
||||||
s := &Scanner{
|
return &Scanner{
|
||||||
files: make([]*FileEntry, 0),
|
files: make([]*FileEntry, 0),
|
||||||
options: opts,
|
options: opts,
|
||||||
fs: fs,
|
fs: fs,
|
||||||
}
|
}
|
||||||
|
|
||||||
// A path that cannot be stat'd, normally because no file is there,
|
|
||||||
// needs no leaving out.
|
|
||||||
for _, p := range opts.ExcludePaths {
|
|
||||||
info, err := s.fs.Stat(p)
|
|
||||||
if err == nil {
|
|
||||||
s.excluded = append(s.excluded, info)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return s
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
|
// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
|
||||||
@@ -450,12 +435,6 @@ func (s *Scanner) enumerateFileWithInfo(
|
|||||||
info = realInfo
|
info = realInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, excluded := range s.excluded {
|
|
||||||
if os.SameFile(info, excluded) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
entry := &FileEntry{
|
entry := &FileEntry{
|
||||||
Path: RelFilePath(cleanPath),
|
Path: RelFilePath(cleanPath),
|
||||||
AbsPath: AbsFilePath(absPath),
|
AbsPath: AbsFilePath(absPath),
|
||||||
|
|||||||
Reference in New Issue
Block a user