Record file mode in the manifest, 0000 unless asked (closes #161)
check / check (push) Waiting to run
check / check (push) Waiting to run
MFFilePath gains mode (field 304): a file's permission bits, 0777 at most, or 0000, meaning none recorded. gen and freshen record real modes only with --include-permissions (ScannerOptions.IncludePermissions); the builder keeps only mode.Perm(), so setuid, setgid and sticky are never written. list -l and export show the mode in octal. check reports MODE_MISMATCH for a recorded mode other than 0000 the file lacks. fetch refuses a manifest with a mode above 0777 before requesting any file, sets only the permission bits of each recorded mode on the files it writes, and downloads again a present file whose mode differs. The decoding-cost bound counts a file entry at 176 bytes, up from 160. Model: opus-5-5
This commit was merged in pull request #163.
This commit is contained in:
@@ -3,6 +3,7 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -374,6 +375,98 @@ func TestGenerateAndCheckCommand(t *testing.T) {
|
||||
assert.Equal(t, 0, exitCode, "check failed: %s", testStderr(t, opts))
|
||||
}
|
||||
|
||||
// TestGenerateRecordsModeOnlyWhenAsked runs gen with and without
|
||||
// --include-permissions: without it every mode is recorded as 0000, with
|
||||
// it each file's permission bits. list -l and export show the recorded
|
||||
// modes.
|
||||
func TestGenerateRecordsModeOnlyWhenAsked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, "/testdir/notes.txt", "hello world")
|
||||
writeTestFile(t, fs, "/testdir/run.sh", "#!/bin/sh\n")
|
||||
require.NoError(t, fs.Chmod("/testdir/run.sh", 0o755))
|
||||
|
||||
for _, tc := range []struct {
|
||||
flags []string
|
||||
want map[string]string // recorded mode by path
|
||||
}{
|
||||
{nil, map[string]string{"notes.txt": "0000", "run.sh": "0000"}},
|
||||
{
|
||||
[]string{"--" + flagIncludePermissions},
|
||||
map[string]string{"notes.txt": "0644", "run.sh": "0755"},
|
||||
},
|
||||
} {
|
||||
opts := testOpts(slices.Concat(
|
||||
[]string{testApp, cmdGenerate, "-q", "-f", "-o", testOutput}, tc.flags,
|
||||
[]string{testDir},
|
||||
), fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
opts = testOpts([]string{testApp, cmdList, "-l", testOutput}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
listed := map[string]string{}
|
||||
out := strings.TrimSuffix(testStdout(t, opts), "\n")
|
||||
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
fields := strings.Split(line, "\t") // mode, size, mtime, path
|
||||
require.Len(t, fields, 4, line)
|
||||
listed[fields[3]] = fields[0]
|
||||
}
|
||||
|
||||
assert.Equal(t, tc.want, listed, "list -l %v", tc.flags)
|
||||
|
||||
opts = testOpts([]string{testApp, cmdExport, testOutput}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
var entries []ExportEntry
|
||||
require.NoError(t, json.Unmarshal([]byte(testStdout(t, opts)), &entries))
|
||||
|
||||
exported := map[string]string{}
|
||||
for _, e := range entries {
|
||||
exported[e.Path] = e.Mode
|
||||
}
|
||||
|
||||
assert.Equal(t, tc.want, exported, "export %v", tc.flags)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckComparesRecordedMode changes a file's mode after gen: check
|
||||
// must fail on it when gen recorded the file's mode, and pass when gen
|
||||
// recorded 0000.
|
||||
func TestCheckComparesRecordedMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
flags []string
|
||||
exitCode int
|
||||
}{
|
||||
{nil, 0},
|
||||
{[]string{"--" + flagIncludePermissions}, 1},
|
||||
} {
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, testFile1, "hello world")
|
||||
|
||||
opts := testOpts(slices.Concat(
|
||||
[]string{testApp, cmdGenerate, "-q", "-o", testMF}, tc.flags,
|
||||
[]string{testDir},
|
||||
), fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
require.NoError(t, fs.Chmod(testFile1, 0o600))
|
||||
|
||||
opts = testOpts([]string{testApp, cmdCheck, testFlagBase, testDir, testMF}, fs)
|
||||
assert.Equal(t, tc.exitCode, runCLI(opts), "%v: %s", tc.flags, testStderr(t, opts))
|
||||
|
||||
if tc.exitCode != 0 {
|
||||
assert.Contains(t, testStderr(t, opts), "MODE_MISMATCH: file1.txt")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sharedWriter appends to a buffer shared with other sharedWriters, so
|
||||
// output written to stdout and stderr is kept in the order it was written.
|
||||
// Each write first waits for delay.
|
||||
|
||||
@@ -162,7 +162,7 @@ func signedManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
|
||||
for path, content := range files {
|
||||
_, err = b.AddFile(mfer.RelFilePath(path), mfer.FileSize(len(content)),
|
||||
mfer.ModTime{}, bytes.NewReader(content), nil)
|
||||
mfer.ModTime{}, 0, bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ type ExportEntry struct {
|
||||
Hashes []string `json:"hashes"`
|
||||
Mtime *string `json:"mtime,omitempty"`
|
||||
Ctime *string `json:"ctime,omitempty"`
|
||||
Mode string `json:"mode"` // octal, "0000" when none was recorded
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) exportManifestOperation(
|
||||
@@ -49,6 +50,7 @@ func (mfa *CLIApp) exportManifestOperation(
|
||||
Path: f.GetPath(),
|
||||
Size: f.GetSize(),
|
||||
Hashes: make([]string, 0, len(f.GetHashes())),
|
||||
Mode: fmt.Sprintf("%04o", f.GetMode()),
|
||||
}
|
||||
|
||||
for _, h := range f.GetHashes() {
|
||||
|
||||
@@ -132,7 +132,7 @@ func TestListFromHTTPURL(t *testing.T) {
|
||||
|
||||
exitCode := runCLI(&RunOptions{
|
||||
Appname: testApp,
|
||||
Args: []string{testApp, "list", server.URL + "/index.mf"},
|
||||
Args: []string{testApp, cmdList, server.URL + "/index.mf"},
|
||||
Stdin: &bytes.Buffer{},
|
||||
Stdout: &stdout,
|
||||
Stderr: &stderr,
|
||||
|
||||
+40
-8
@@ -95,6 +95,9 @@ var (
|
||||
// writes another file.
|
||||
errNameClash = errors.New(
|
||||
"manifest lists a file where fetch writes another file")
|
||||
// errModeOutOfRange indicates a manifest that lists a mode with more
|
||||
// than the permission bits, such as setuid.
|
||||
errModeOutOfRange = errors.New("manifest lists a mode outside 0777")
|
||||
)
|
||||
|
||||
// DownloadProgress reports the progress of a single file download.
|
||||
@@ -292,11 +295,11 @@ func downloadManifestFiles(
|
||||
}
|
||||
|
||||
// alreadyPresent reports whether localPath under dest is a regular file
|
||||
// with the size and one of the hashes the manifest lists for entry. It
|
||||
// hashes the whole file, since a matching size alone would accept a
|
||||
// corrupted or partly written one. A file it cannot read, or reaches only
|
||||
// through a symlink, is not present: fetch downloads it, and the download
|
||||
// reports the problem.
|
||||
// with the size, the recorded mode if any, and one of the hashes the
|
||||
// manifest lists for entry. It hashes the whole file, since a matching
|
||||
// size alone would accept a corrupted or partly written one. A file it
|
||||
// cannot read, or reaches only through a symlink, is not present: fetch
|
||||
// downloads it, and the download reports the problem.
|
||||
func alreadyPresent(dest, localPath string, entry *mfer.MFFilePath) bool {
|
||||
if checkNoSymlinks(dest, localPath) != nil {
|
||||
return false
|
||||
@@ -309,6 +312,12 @@ func alreadyPresent(dest, localPath string, entry *mfer.MFFilePath) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// A recorded mode of 0 means none was recorded.
|
||||
if entry.GetMode() != 0 &&
|
||||
info.Mode().Perm() != os.FileMode(entry.GetMode()).Perm() {
|
||||
return false
|
||||
}
|
||||
|
||||
// G304: localPath is a relative path that sanitizePath keeps inside
|
||||
// dest as text, and checkNoSymlinks just found no symlink in it.
|
||||
f, err := os.Open(path) //nolint:gosec // G304: see comment above
|
||||
@@ -414,9 +423,9 @@ func (mfa *CLIApp) fetchManifestOperation(
|
||||
|
||||
// fetchManifest downloads the manifest at manifestURL and parses it,
|
||||
// enforcing --require-signature if it is given and refusing a manifest
|
||||
// that lists a file where fetch writes another. It returns the manifest as
|
||||
// downloaded, to be saved once the files are in place, and the files it
|
||||
// lists.
|
||||
// that lists a file where fetch writes another or a mode outside 0777. It
|
||||
// returns the manifest as downloaded, to be saved once the files are in
|
||||
// place, and the files it lists.
|
||||
func fetchManifest(
|
||||
ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
|
||||
) ([]byte, []*mfer.MFFilePath, error) {
|
||||
@@ -460,6 +469,16 @@ func fetchManifest(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// fetch sets each recorded mode on the file it writes, so a mode above
|
||||
// 0777, which could carry setuid, setgid or sticky bits, is refused
|
||||
// before any file is requested.
|
||||
for _, f := range files {
|
||||
if f.GetMode() > uint32(os.ModePerm) {
|
||||
return nil, nil, fmt.Errorf("%w: %s (%#o)",
|
||||
errModeOutOfRange, f.GetPath(), f.GetMode())
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("manifest contains %d files", len(files))
|
||||
|
||||
return manifestData, files, nil
|
||||
@@ -866,6 +885,19 @@ func saveResponse(
|
||||
return err
|
||||
}
|
||||
|
||||
// A recorded mode of 0 means none was recorded. Of any other, only the
|
||||
// permission bits are set, whatever the umask, so setuid, setgid and
|
||||
// sticky never are, whichever caller passed the entry.
|
||||
if entry.GetMode() != 0 {
|
||||
err = out.Chmod(os.FileMode(entry.GetMode()).Perm())
|
||||
if err != nil {
|
||||
_ = out.Close()
|
||||
_ = os.Remove(filepath.Join(dest, tmpPath))
|
||||
|
||||
return fmt.Errorf("failed to set mode: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set up hash computation
|
||||
h := sha256.New()
|
||||
|
||||
|
||||
+193
-1
@@ -4,6 +4,7 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
@@ -19,9 +20,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
@@ -1299,6 +1304,193 @@ func TestFetchRefusesNamesEqualIgnoringCase(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestFetchSetsRecordedMode fetches a tree whose manifest records the
|
||||
// modes 0640 and 0755. Each file must get its mode whatever the umask, and
|
||||
// check must pass on the result. After one file's mode is changed, a
|
||||
// second fetch must download that file again, and only it, to restore its
|
||||
// mode.
|
||||
func TestFetchSetsRecordedMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
files := map[string][]byte{
|
||||
testFileTxt: []byte("a file"),
|
||||
"tool.sh": []byte("#!/bin/sh\n"),
|
||||
}
|
||||
modes := map[string]os.FileMode{testFileTxt: 0o640, "tool.sh": 0o755}
|
||||
|
||||
sourceFs := afero.NewMemMapFs()
|
||||
for p, content := range files {
|
||||
require.NoError(t, afero.WriteFile(sourceFs, "/"+p, content, modes[p]))
|
||||
}
|
||||
|
||||
scanner := mfer.NewScannerWithOptions(&mfer.ScannerOptions{
|
||||
Fs: sourceFs,
|
||||
IncludePermissions: true,
|
||||
})
|
||||
require.NoError(t, scanner.EnumerateFS(sourceFs, "/", nil))
|
||||
|
||||
var manifest bytes.Buffer
|
||||
require.NoError(t, scanner.ToManifest(context.Background(), &manifest, nil))
|
||||
|
||||
tree := fetchTestHandler(manifest.Bytes(), files)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
requested []string
|
||||
)
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
|
||||
requested = append(requested, r.URL.Path)
|
||||
|
||||
mu.Unlock()
|
||||
|
||||
tree.ServeHTTP(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
dest := t.TempDir()
|
||||
fetch := []string{testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL}
|
||||
|
||||
opts := testOpts(fetch, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
for p, mode := range modes {
|
||||
info, err := os.Stat(filepath.Join(dest, p))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, mode, info.Mode().Perm(), p)
|
||||
}
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, cmdCheck, "-q", testFlagBase, dest,
|
||||
filepath.Join(dest, defaultManifestName),
|
||||
}, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
require.NoError(t, os.Chmod(filepath.Join(dest, testFileTxt), 0o600))
|
||||
|
||||
mu.Lock()
|
||||
requested = nil
|
||||
mu.Unlock()
|
||||
|
||||
opts = testOpts(fetch, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
info, err := os.Stat(filepath.Join(dest, testFileTxt))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o640), info.Mode().Perm())
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
assert.ElementsMatch(t,
|
||||
[]string{"/" + defaultManifestName, "/" + testFileTxt}, requested)
|
||||
}
|
||||
|
||||
// TestFetchRefusesModeOutsidePermissionBits fetches manifests that record
|
||||
// a mode with the setuid bit, once as Unix writes it (04755) and once as
|
||||
// Go keeps it in os.FileMode. fetch must refuse both before it creates the
|
||||
// destination or requests any file.
|
||||
func TestFetchRefusesModeOutsidePermissionBits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
files := map[string][]byte{testFileTxt: []byte("a file")}
|
||||
|
||||
assertFetchRefused(t, manifestWithMode(t, testFileTxt, files[testFileTxt], 0o4755),
|
||||
files, "manifest lists a mode outside 0777: file.txt (04755)")
|
||||
|
||||
assertFetchRefused(t,
|
||||
manifestWithMode(t, testFileTxt, files[testFileTxt],
|
||||
uint32(os.ModeSetuid|0o755)),
|
||||
files, "manifest lists a mode outside 0777: file.txt (040000755)")
|
||||
}
|
||||
|
||||
// TestDownloadFileSetsOnlyPermissionBits downloads a file whose entry
|
||||
// records mode 0755 together with setuid, setgid or sticky, as Go keeps
|
||||
// them in os.FileMode. fetchManifest would refuse such an entry; called
|
||||
// directly, downloadFile must still set only the permission bits.
|
||||
func TestDownloadFileSetsOnlyPermissionBits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
content := []byte("#!/bin/sh\n")
|
||||
|
||||
digest := sha256.Sum256(content)
|
||||
hash, err := multihash.Encode(digest[:], multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(content)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
for _, special := range []os.FileMode{os.ModeSetuid, os.ModeSetgid, os.ModeSticky} {
|
||||
entry := &mfer.MFFilePath{
|
||||
Path: testFileTxt,
|
||||
Size: int64(len(content)),
|
||||
Hashes: []*mfer.MFFileChecksum{{MultiHash: hash}},
|
||||
Mode: uint32(special | 0o755),
|
||||
}
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
err := downloadFile(context.Background(), testClient(),
|
||||
server.URL+"/"+testFileTxt, dest, testFileTxt, entry, nil)
|
||||
require.NoError(t, err, special)
|
||||
|
||||
info, err := os.Stat(filepath.Join(dest, testFileTxt))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode(), special)
|
||||
}
|
||||
}
|
||||
|
||||
// manifestWithMode returns a manifest listing one file, path, with content
|
||||
// and the given recorded mode. It is assembled by hand, since the builder
|
||||
// never records a mode outside 0777.
|
||||
func manifestWithMode(t *testing.T, path string, content []byte, mode uint32) []byte {
|
||||
t.Helper()
|
||||
|
||||
digest := sha256.Sum256(content)
|
||||
hash, err := multihash.Encode(digest[:], multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
id := uuid.New()
|
||||
|
||||
inner, err := proto.Marshal(&mfer.MFFile{
|
||||
Version: mfer.MFFile_VERSION_ONE,
|
||||
Files: []*mfer.MFFilePath{{
|
||||
Path: path,
|
||||
Size: int64(len(content)),
|
||||
Hashes: []*mfer.MFFileChecksum{{MultiHash: hash}},
|
||||
Mode: mode,
|
||||
}},
|
||||
Uuid: id[:],
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
encoder, err := zstd.NewWriter(nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
compressed := encoder.EncodeAll(inner, nil)
|
||||
require.NoError(t, encoder.Close())
|
||||
|
||||
sum := sha256.Sum256(compressed)
|
||||
|
||||
outer, err := proto.Marshal(&mfer.MFFileOuter{
|
||||
Version: mfer.MFFileOuter_VERSION_ONE,
|
||||
CompressionType: mfer.MFFileOuter_COMPRESSION_ZSTD,
|
||||
Size: int64(len(inner)),
|
||||
Sha256: sum[:],
|
||||
Uuid: id[:],
|
||||
InnerMessage: compressed,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return append([]byte(mfer.MAGIC), outer...)
|
||||
}
|
||||
|
||||
// builtManifest returns a manifest of files, built directly rather than
|
||||
// scanned, since a scan lists no hidden files and never a path starting
|
||||
// with "./".
|
||||
@@ -1309,7 +1501,7 @@ func builtManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
|
||||
for p, content := range files {
|
||||
_, err := builder.AddFile(mfer.RelFilePath(p), mfer.FileSize(len(content)),
|
||||
mfer.ModTime(time.Now()), bytes.NewReader(content), nil)
|
||||
mfer.ModTime(time.Now()), 0, bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
|
||||
+38
-23
@@ -48,6 +48,7 @@ type freshenEntry struct {
|
||||
path string
|
||||
size int64
|
||||
mtime time.Time
|
||||
mode fs.FileMode // mode to record, 0 for none
|
||||
needsHash bool // true if new or changed
|
||||
existing *mfer.MFFilePath // existing manifest entry if unchanged
|
||||
}
|
||||
@@ -55,13 +56,14 @@ type freshenEntry struct {
|
||||
// freshenScanner walks the filesystem and compares it against the
|
||||
// entries of an existing manifest.
|
||||
type freshenScanner struct {
|
||||
fs afero.Fs
|
||||
absBase string
|
||||
excluded []fs.FileInfo // files left out of the listing
|
||||
includeDotfiles bool
|
||||
followSymlinks bool
|
||||
showProgress bool
|
||||
existingByPath map[string]*mfer.MFFilePath
|
||||
fs afero.Fs
|
||||
absBase string
|
||||
excluded []fs.FileInfo // files left out of the listing
|
||||
includeDotfiles bool
|
||||
followSymlinks bool
|
||||
includePermissions bool
|
||||
showProgress bool
|
||||
existingByPath map[string]*mfer.MFFilePath
|
||||
|
||||
entries []*freshenEntry
|
||||
scanCount int64
|
||||
@@ -93,6 +95,12 @@ func (s *freshenScanner) resolveSymlink(path string) (fs.FileInfo, bool) {
|
||||
// recordEntry classifies a scanned file as changed, unchanged, or added
|
||||
// relative to the existing manifest.
|
||||
func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
// A mode of 0 records 0000, which means none was recorded.
|
||||
var mode fs.FileMode
|
||||
if s.includePermissions {
|
||||
mode = info.Mode().Perm()
|
||||
}
|
||||
|
||||
existing, inManifest := s.existingByPath[relPath]
|
||||
if !inManifest {
|
||||
s.added++
|
||||
@@ -102,16 +110,17 @@ func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
mode: mode,
|
||||
needsHash: true,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Check if changed (size or mtime). An entry with no recorded mtime
|
||||
// cannot be compared, so it counts as changed and gets re-hashed;
|
||||
// silently treating the absent mtime as the Unix epoch would classify
|
||||
// every such entry as changed without saying why.
|
||||
// Check if changed (size, mtime, or the mode to record). An entry
|
||||
// with no recorded mtime cannot be compared, so it counts as changed
|
||||
// and gets re-hashed; silently treating the absent mtime as the Unix
|
||||
// epoch would classify every such entry as changed without saying why.
|
||||
existingMtime, haveMtime := entryMtime(existing)
|
||||
if !haveMtime {
|
||||
log.Debugf("%s: manifest entry has no mtime, treating as changed",
|
||||
@@ -119,7 +128,8 @@ func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
}
|
||||
|
||||
if !haveMtime || existing.GetSize() != info.Size() ||
|
||||
!existingMtime.Equal(info.ModTime()) {
|
||||
!existingMtime.Equal(info.ModTime()) ||
|
||||
fs.FileMode(existing.GetMode()) != mode {
|
||||
s.changed++
|
||||
|
||||
log.Verbosef("M %s", relPath)
|
||||
@@ -127,6 +137,7 @@ func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
mode: mode,
|
||||
needsHash: true,
|
||||
})
|
||||
} else {
|
||||
@@ -136,6 +147,7 @@ func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
mode: mode,
|
||||
needsHash: false,
|
||||
existing: existing,
|
||||
})
|
||||
@@ -296,7 +308,7 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
||||
h.hashedFiles++
|
||||
|
||||
// Add to builder with computed hash
|
||||
err = addFileToBuilder(h.builder, e.path, e.size, e.mtime, hash)
|
||||
err = addFileToBuilder(h.builder, e.path, e.size, e.mtime, e.mode, hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
||||
}
|
||||
@@ -379,13 +391,14 @@ func (mfa *CLIApp) freshenScan(
|
||||
}
|
||||
|
||||
scanner := &freshenScanner{
|
||||
fs: mfa.Fs,
|
||||
absBase: absBase,
|
||||
excluded: excluded,
|
||||
includeDotfiles: cmd.Bool("include-dotfiles"),
|
||||
followSymlinks: cmd.Bool("follow-symlinks"),
|
||||
showProgress: showProgress,
|
||||
existingByPath: existingByPath,
|
||||
fs: mfa.Fs,
|
||||
absBase: absBase,
|
||||
excluded: excluded,
|
||||
includeDotfiles: cmd.Bool("include-dotfiles"),
|
||||
followSymlinks: cmd.Bool("follow-symlinks"),
|
||||
includePermissions: cmd.Bool(flagIncludePermissions),
|
||||
showProgress: showProgress,
|
||||
existingByPath: existingByPath,
|
||||
}
|
||||
|
||||
err := afero.Walk(mfa.Fs, absBase, scanner.walk)
|
||||
@@ -611,10 +624,12 @@ func hashFile(r io.Reader, progress func(int64)) ([]byte, int64, error) {
|
||||
|
||||
// addFileToBuilder adds a new file entry to the builder
|
||||
func addFileToBuilder(
|
||||
b *mfer.Builder, path string, size int64, mtime time.Time, hash []byte,
|
||||
b *mfer.Builder, path string, size int64, mtime time.Time, mode fs.FileMode,
|
||||
hash []byte,
|
||||
) error {
|
||||
return b.AddFileWithHash(
|
||||
mfer.RelFilePath(path), mfer.FileSize(size), mfer.ModTime(mtime), hash)
|
||||
mfer.RelFilePath(path), mfer.FileSize(size), mfer.ModTime(mtime), mode,
|
||||
hash)
|
||||
}
|
||||
|
||||
// addExistingToBuilder adds an existing manifest entry to the builder.
|
||||
@@ -634,7 +649,7 @@ func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
||||
|
||||
err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
||||
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
||||
entry.GetHashes()[0].GetMultiHash())
|
||||
fs.FileMode(entry.GetMode()), entry.GetHashes()[0].GetMultiHash())
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"manifest entry %s: %w (regenerate the manifest with mfer generate)",
|
||||
|
||||
@@ -99,6 +99,42 @@ func TestFreshenUnchanged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFreshenRecordsModeOnlyWhenAsked changes only the modes of a tree
|
||||
// after gen made its manifest, which recorded every mode as 0000. freshen
|
||||
// --include-permissions must record each file's permission bits, and a
|
||||
// later freshen without it must record 0000 again, although no file's
|
||||
// content or mtime changed.
|
||||
func TestFreshenRecordsModeOnlyWhenAsked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
root, manifestPath := setupFreshenDir(t, fs,
|
||||
map[string]string{testFileTxt: "a file", testDirFile: "a file in dir"})
|
||||
require.NoError(t, fs.Chmod(filepath.Join(root, testFileTxt), 0o640))
|
||||
require.NoError(t, fs.Chmod(filepath.Join(root, testDirFile), 0o755))
|
||||
|
||||
recordedModes := func() map[string]uint32 {
|
||||
modes := map[string]uint32{}
|
||||
for _, f := range manifestFiles(t, fs, manifestPath) {
|
||||
modes[f.GetPath()] = f.GetMode()
|
||||
}
|
||||
|
||||
return modes
|
||||
}
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFreshen, "-q", "--" + flagIncludePermissions,
|
||||
testFlagBase, root, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
assert.Equal(t, map[string]uint32{testFileTxt: 0o640, testDirFile: 0o755},
|
||||
recordedModes())
|
||||
|
||||
runFreshen(t, fs, root, manifestPath)
|
||||
assert.Equal(t, map[string]uint32{testFileTxt: 0, testDirFile: 0},
|
||||
recordedModes())
|
||||
}
|
||||
|
||||
// assertManifestLists asserts that the manifest at manifestPath lists
|
||||
// exactly the files in want, each with the size and SHA-256 hash of its
|
||||
// content in want and the mtime of the file of that name under root.
|
||||
|
||||
+5
-4
@@ -92,10 +92,11 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
||||
func (mfa *CLIApp) buildScannerOptions(cmd *cli.Command) *mfer.ScannerOptions {
|
||||
output := cmd.String("output")
|
||||
opts := &mfer.ScannerOptions{
|
||||
IncludeDotfiles: cmd.Bool("include-dotfiles"),
|
||||
FollowSymLinks: cmd.Bool("follow-symlinks"),
|
||||
IncludeTimestamps: cmd.Bool("include-timestamps"),
|
||||
Fs: mfa.Fs,
|
||||
IncludeDotfiles: cmd.Bool("include-dotfiles"),
|
||||
FollowSymLinks: cmd.Bool("follow-symlinks"),
|
||||
IncludeTimestamps: cmd.Bool("include-timestamps"),
|
||||
IncludePermissions: cmd.Bool(flagIncludePermissions),
|
||||
Fs: mfa.Fs,
|
||||
// Neither a manifest being replaced nor a temp file left by an
|
||||
// interrupted run belongs in the new manifest.
|
||||
ExcludePaths: []string{output, manifestTempPath(output)},
|
||||
|
||||
@@ -52,8 +52,8 @@ func (mfa *CLIApp) listManifestOperation(ctx context.Context, cmd *cli.Command)
|
||||
mtimeStr = mtime.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(mfa.Stdout, "%d\t%s\t%s%s",
|
||||
f.GetSize(), mtimeStr, f.GetPath(), lineEnd)
|
||||
_, _ = fmt.Fprintf(mfa.Stdout, "%04o\t%d\t%s\t%s%s",
|
||||
f.GetMode(), f.GetSize(), mtimeStr, f.GetPath(), lineEnd)
|
||||
} else {
|
||||
_, _ = fmt.Fprintf(mfa.Stdout, "%s%s", f.GetPath(), lineEnd)
|
||||
}
|
||||
|
||||
+20
-6
@@ -21,12 +21,14 @@ const (
|
||||
cmdFreshen = "freshen"
|
||||
cmdExport = "export"
|
||||
cmdFetch = "fetch"
|
||||
cmdList = "list"
|
||||
cmdVersion = "version"
|
||||
|
||||
flagProgress = "progress"
|
||||
flagTimeout = "timeout"
|
||||
flagDest = "dest"
|
||||
flagRequireSignature = "require-signature"
|
||||
flagProgress = "progress"
|
||||
flagTimeout = "timeout"
|
||||
flagDest = "dest"
|
||||
flagRequireSignature = "require-signature"
|
||||
flagIncludePermissions = "include-permissions"
|
||||
|
||||
manifestArgsUsage = "[manifest file]"
|
||||
|
||||
@@ -167,6 +169,16 @@ func requireSignatureFlag() *cli.StringFlag {
|
||||
}
|
||||
}
|
||||
|
||||
// includePermissionsFlag returns the --include-permissions flag taken by the
|
||||
// generate and freshen subcommands.
|
||||
func includePermissionsFlag() *cli.BoolFlag {
|
||||
return &cli.BoolFlag{
|
||||
Name: flagIncludePermissions,
|
||||
Usage: "Record each file's permission bits in manifest " +
|
||||
"(recorded as 0000 by default)",
|
||||
}
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdGenerate,
|
||||
@@ -224,6 +236,7 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
Usage: "Include createdAt timestamp in manifest " +
|
||||
"(omitted by default for determinism)",
|
||||
},
|
||||
includePermissionsFlag(),
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -307,6 +320,7 @@ func (mfa *CLIApp) freshenCommand() *cli.Command {
|
||||
Usage: "Include createdAt timestamp in manifest " +
|
||||
"(omitted by default for determinism)",
|
||||
},
|
||||
includePermissionsFlag(),
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -340,7 +354,7 @@ func (mfa *CLIApp) versionCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) listCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "list",
|
||||
Name: cmdList,
|
||||
Aliases: []string{"ls"},
|
||||
Usage: "List files in manifest",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
@@ -350,7 +364,7 @@ func (mfa *CLIApp) listCommand() *cli.Command {
|
||||
&cli.BoolFlag{
|
||||
Name: "long",
|
||||
Aliases: []string{"l"},
|
||||
Usage: "Show size and mtime",
|
||||
Usage: "Show mode, size and mtime",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "print0",
|
||||
|
||||
Reference in New Issue
Block a user