Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0afcbae783 |
@@ -28,7 +28,6 @@ const (
|
|||||||
testFile1 = "/testdir/file1.txt"
|
testFile1 = "/testdir/file1.txt"
|
||||||
testMF = "/testdir/test.mf"
|
testMF = "/testdir/test.mf"
|
||||||
testOutput = "/output.mf"
|
testOutput = "/output.mf"
|
||||||
testOutputTmp = "/output.mf.tmp"
|
|
||||||
testManifest = "/manifest.mf"
|
testManifest = "/manifest.mf"
|
||||||
testFlagBase = "--base"
|
testFlagBase = "--base"
|
||||||
testFlagNoExtra = "--no-extra-files"
|
testFlagNoExtra = "--no-extra-files"
|
||||||
@@ -683,7 +682,7 @@ func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
|
|||||||
assert.True(t, exists, "output file should exist")
|
assert.True(t, exists, "output file should exist")
|
||||||
|
|
||||||
// Verify temp file does NOT exist
|
// Verify temp file does NOT exist
|
||||||
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.False(t, tmpExists,
|
assert.False(t, tmpExists,
|
||||||
"temp file should not exist after successful generation")
|
"temp file should not exist after successful generation")
|
||||||
@@ -715,7 +714,7 @@ func TestGenerateAtomicWriteOverwriteWithForce(t *testing.T) {
|
|||||||
"manifest should be overwritten")
|
"manifest should be overwritten")
|
||||||
|
|
||||||
// Verify temp file does NOT exist
|
// Verify temp file does NOT exist
|
||||||
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.False(t, tmpExists,
|
assert.False(t, tmpExists,
|
||||||
"temp file should not exist after successful generation")
|
"temp file should not exist after successful generation")
|
||||||
@@ -822,9 +821,8 @@ func TestGenerateDefaultOutputLeftOutOfListing(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestGenerateLeavesLeftoverTempFileOutOfListing runs gen where an
|
// TestGenerateLeavesLeftoverTempFileOutOfListing runs gen where an
|
||||||
// interrupted run left its temp file, index.mf.tmp, beside the output:
|
// interrupted run left its temp file beside the output: the leftover is
|
||||||
// the leftover is not listed, and gen does not fail when it overwrites
|
// not listed, and gen does not fail when it overwrites it.
|
||||||
// it.
|
|
||||||
func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -833,7 +831,7 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
|||||||
|
|
||||||
fs := afero.NewOsFs()
|
fs := afero.NewOsFs()
|
||||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
||||||
writeTestFile(t, fs, output+".tmp", "part of a manifest")
|
writeTestFile(t, fs, manifestTempPath(output), "part of a manifest")
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", output, root}, fs)
|
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", output, root}, fs)
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||||
@@ -863,7 +861,7 @@ func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
|||||||
exists, _ := afero.Exists(fs, testOutput)
|
exists, _ := afero.Exists(fs, testOutput)
|
||||||
assert.True(t, exists, "output file should exist")
|
assert.True(t, exists, "output file should exist")
|
||||||
|
|
||||||
tmpExists, _ := afero.Exists(fs, testOutputTmp)
|
tmpExists, _ := afero.Exists(fs, manifestTempPath(testOutput))
|
||||||
assert.False(t, tmpExists, "temp file should be cleaned up")
|
assert.False(t, tmpExists, "temp file should be cleaned up")
|
||||||
|
|
||||||
// Verify manifest is valid (not empty)
|
// Verify manifest is valid (not empty)
|
||||||
@@ -929,7 +927,7 @@ func TestGenerateAtomicWriteCleansUpOnError(t *testing.T) {
|
|||||||
"output file should not exist after failed generation (atomic write)")
|
"output file should not exist after failed generation (atomic write)")
|
||||||
|
|
||||||
// Temp file should also not exist
|
// Temp file should also not exist
|
||||||
tmpExists, _ := afero.Exists(baseFs, testOutputTmp)
|
tmpExists, _ := afero.Exists(baseFs, manifestTempPath(testOutput))
|
||||||
assert.False(t, tmpExists,
|
assert.False(t, tmpExists,
|
||||||
"temp file should be cleaned up after failed generation")
|
"temp file should be cleaned up after failed generation")
|
||||||
}
|
}
|
||||||
|
|||||||
+17
-23
@@ -57,8 +57,7 @@ type freshenEntry struct {
|
|||||||
type freshenScanner struct {
|
type freshenScanner struct {
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
absBase string
|
absBase string
|
||||||
manifestInfo fs.FileInfo // the manifest being freshened
|
excluded []fs.FileInfo // files left out of the listing
|
||||||
tmpInfo fs.FileInfo // its leftover temp file; nil when there is none
|
|
||||||
includeDotfiles bool
|
includeDotfiles bool
|
||||||
followSymlinks bool
|
followSymlinks bool
|
||||||
showProgress bool
|
showProgress bool
|
||||||
@@ -182,11 +181,10 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
|||||||
info = realInfo
|
info = realInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip the manifest file itself and a temp file left beside it by an
|
for _, excluded := range s.excluded {
|
||||||
// interrupted run, however their paths are spelled; gen leaves out its
|
if os.SameFile(info, excluded) {
|
||||||
// output file the same way. os.SameFile never matches a nil FileInfo.
|
return nil
|
||||||
if os.SameFile(info, s.manifestInfo) || os.SameFile(info, s.tmpInfo) {
|
}
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
s.scanCount++
|
s.scanCount++
|
||||||
@@ -311,7 +309,7 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
|||||||
func writeFreshenedManifest(
|
func writeFreshenedManifest(
|
||||||
ctx context.Context, afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
ctx context.Context, afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
||||||
) error {
|
) error {
|
||||||
tmpPath := manifestPath + ".tmp"
|
tmpPath := manifestTempPath(manifestPath)
|
||||||
|
|
||||||
outFile, err := afs.Create(tmpPath)
|
outFile, err := afs.Create(tmpPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -368,33 +366,29 @@ func (mfa *CLIApp) freshenScan(
|
|||||||
startScan := time.Now()
|
startScan := time.Now()
|
||||||
showProgress := ctx.Bool("progress")
|
showProgress := ctx.Bool("progress")
|
||||||
|
|
||||||
manifestInfo, err := mfa.Fs.Stat(manifestPath)
|
// Leave out the manifest and a temp file left by an interrupted run,
|
||||||
if err != nil {
|
// as gen does. A path that cannot be stat'd, normally because no file
|
||||||
return nil, 0, fmt.Errorf("freshen: %w", err)
|
// is there, needs no leaving out.
|
||||||
}
|
var excluded []fs.FileInfo
|
||||||
|
|
||||||
// The temp file writeFreshenedManifest writes, if an interrupted run
|
for _, p := range []string{manifestPath, manifestTempPath(manifestPath)} {
|
||||||
// left one. One that cannot be stat'd, normally because it does not
|
info, err := mfa.Fs.Stat(p)
|
||||||
// exist, needs no leaving out.
|
if err == nil {
|
||||||
var tmpInfo fs.FileInfo
|
excluded = append(excluded, info)
|
||||||
|
}
|
||||||
info, err := mfa.Fs.Stat(manifestPath + ".tmp")
|
|
||||||
if err == nil {
|
|
||||||
tmpInfo = info
|
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := &freshenScanner{
|
scanner := &freshenScanner{
|
||||||
fs: mfa.Fs,
|
fs: mfa.Fs,
|
||||||
absBase: absBase,
|
absBase: absBase,
|
||||||
manifestInfo: manifestInfo,
|
excluded: excluded,
|
||||||
tmpInfo: tmpInfo,
|
|
||||||
includeDotfiles: ctx.Bool("include-dotfiles"),
|
includeDotfiles: ctx.Bool("include-dotfiles"),
|
||||||
followSymlinks: ctx.Bool("follow-symlinks"),
|
followSymlinks: ctx.Bool("follow-symlinks"),
|
||||||
showProgress: showProgress,
|
showProgress: showProgress,
|
||||||
existingByPath: existingByPath,
|
existingByPath: existingByPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = afero.Walk(mfa.Fs, absBase, scanner.walk)
|
err := afero.Walk(mfa.Fs, absBase, scanner.walk)
|
||||||
|
|
||||||
if showProgress {
|
if showProgress {
|
||||||
log.ProgressDone()
|
log.ProgressDone()
|
||||||
|
|||||||
@@ -133,8 +133,8 @@ func TestFreshenLeavesManifestOutOfListing(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestFreshenLeavesLeftoverTempFileOutOfListing freshens a manifest where
|
// TestFreshenLeavesLeftoverTempFileOutOfListing freshens a manifest where
|
||||||
// an interrupted run left its temp file, index.mf.tmp, beside it: the
|
// an interrupted run left its temp file beside it: the leftover is not
|
||||||
// leftover is not listed.
|
// listed.
|
||||||
func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -147,7 +147,7 @@ func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
|||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||||
|
|
||||||
writeTestFile(t, fs, manifestPath+".tmp", "part of a manifest")
|
writeTestFile(t, fs, manifestTempPath(manifestPath), "part of a manifest")
|
||||||
// A new file gives freshen something to write.
|
// A new file gives freshen something to write.
|
||||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
||||||
|
|
||||||
|
|||||||
+5
-2
@@ -89,12 +89,15 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
|||||||
|
|
||||||
// buildScannerOptions constructs scanner options from the CLI flags.
|
// buildScannerOptions constructs scanner options from the CLI flags.
|
||||||
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
||||||
|
output := ctx.String("output")
|
||||||
opts := &mfer.ScannerOptions{
|
opts := &mfer.ScannerOptions{
|
||||||
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
||||||
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
||||||
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
||||||
Fs: mfa.Fs,
|
Fs: mfa.Fs,
|
||||||
OutputPath: ctx.String("output"),
|
// Neither a manifest being replaced nor a temp file left by an
|
||||||
|
// interrupted run belongs in the new manifest.
|
||||||
|
ExcludePaths: []string{output, manifestTempPath(output)},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set seed for deterministic UUID if provided
|
// Set seed for deterministic UUID if provided
|
||||||
@@ -218,7 +221,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Create temp file for atomic write
|
// Create temp file for atomic write
|
||||||
tmpPath := outputPath + ".tmp"
|
tmpPath := manifestTempPath(outputPath)
|
||||||
|
|
||||||
outFile, err := mfa.Fs.Create(tmpPath)
|
outFile, err := mfa.Fs.Create(tmpPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -32,6 +32,12 @@ const (
|
|||||||
defaultManifestName = "index.mf"
|
defaultManifestName = "index.mf"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// manifestTempPath returns the temp file gen and freshen write a manifest
|
||||||
|
// to before renaming it to out.
|
||||||
|
func manifestTempPath(out string) string {
|
||||||
|
return out + ".tmp"
|
||||||
|
}
|
||||||
|
|
||||||
// errUnknownCommand indicates an unrecognized command argument.
|
// errUnknownCommand indicates an unrecognized command argument.
|
||||||
var errUnknownCommand = errors.New("unknown command")
|
var errUnknownCommand = errors.New("unknown command")
|
||||||
|
|
||||||
|
|||||||
+12
-25
@@ -58,15 +58,8 @@ type ScannerOptions struct {
|
|||||||
SigningOptions *SigningOptions
|
SigningOptions *SigningOptions
|
||||||
// Seed, if set, derives a deterministic UUID from this seed.
|
// Seed, if set, derives a deterministic UUID from this seed.
|
||||||
Seed string
|
Seed string
|
||||||
// OutputPath, if set, is the file the manifest will be written to.
|
// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
|
||||||
// Enumeration leaves out that file and OutputPath + ".tmp", the temp
|
ExcludePaths []string
|
||||||
// file mfer gen writes first, if they exist when the scanner is
|
|
||||||
// created, however their paths are spelled (through a symlink, a hard
|
|
||||||
// link, or in different letter case), so neither a manifest being
|
|
||||||
// replaced nor a temp file left by an interrupted run is ever listed.
|
|
||||||
// Files are matched with os.SameFile, which only recognizes files on
|
|
||||||
// the operating system's filesystem.
|
|
||||||
OutputPath string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FileEntry represents a file that has been enumerated.
|
// FileEntry represents a file that has been enumerated.
|
||||||
@@ -85,8 +78,7 @@ type Scanner struct {
|
|||||||
totalBytes FileSize // cached sum of all file sizes
|
totalBytes FileSize // cached sum of all file sizes
|
||||||
options *ScannerOptions
|
options *ScannerOptions
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
outputInfo fs.FileInfo // the existing output file; nil when there is none
|
excluded []fs.FileInfo // the files named in ExcludePaths that exist
|
||||||
tmpInfo fs.FileInfo // its leftover temp file; nil when there is none
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewScanner creates a new Scanner with default options.
|
// NewScanner creates a new Scanner with default options.
|
||||||
@@ -111,17 +103,12 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
|
|||||||
fs: fs,
|
fs: fs,
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.OutputPath != "" {
|
// A path that cannot be stat'd, normally because no file is there,
|
||||||
// A file that cannot be stat'd, normally because it does not
|
// needs no leaving out.
|
||||||
// exist, needs no leaving out.
|
for _, p := range opts.ExcludePaths {
|
||||||
info, err := s.fs.Stat(opts.OutputPath)
|
info, err := s.fs.Stat(p)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
s.outputInfo = info
|
s.excluded = append(s.excluded, info)
|
||||||
}
|
|
||||||
|
|
||||||
info, err = s.fs.Stat(opts.OutputPath + ".tmp")
|
|
||||||
if err == nil {
|
|
||||||
s.tmpInfo = info
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -463,10 +450,10 @@ func (s *Scanner) enumerateFileWithInfo(
|
|||||||
info = realInfo
|
info = realInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
// Neither the manifest being written nor its temp file is one of the
|
for _, excluded := range s.excluded {
|
||||||
// files it lists. os.SameFile never matches a nil FileInfo.
|
if os.SameFile(info, excluded) {
|
||||||
if os.SameFile(info, s.outputInfo) || os.SameFile(info, s.tmpInfo) {
|
return nil
|
||||||
return nil
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
entry := &FileEntry{
|
entry := &FileEntry{
|
||||||
|
|||||||
Reference in New Issue
Block a user