Record file mode in the manifest, 0000 unless asked (closes #161)

MFFilePath gains mode (field 304): a file's permission bits, 0777 at
most, or 0000, meaning none recorded. gen and freshen record real modes
only with --include-permissions (ScannerOptions.IncludePermissions); the
builder keeps only mode.Perm(), so setuid, setgid and sticky are never
written. list -l and export show the mode in octal. check reports
MODE_MISMATCH for a recorded mode other than 0000 the file lacks. fetch
refuses a manifest with a mode above 0777 before requesting any file,
sets only the permission bits of each recorded mode on the files it
writes, and downloads again a present file whose mode differs. The decoding-cost bound counts a file
entry at 176 bytes, up from 160.

Model: opus-5-5

internal/cli/fetch_test.go

@@ -4,6 +4,7 @@ package cli
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"maps"
@@ -19,9 +20,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/klauspost/compress/zstd"
+	"github.com/multiformats/go-multihash"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
 	"sneak.berlin/go/mfer/mfer"
 )
 
@@ -1299,6 +1304,193 @@ func TestFetchRefusesNamesEqualIgnoringCase(t *testing.T) {
 	})
 }
 
+// TestFetchSetsRecordedMode fetches a tree whose manifest records the
+// modes 0640 and 0755. Each file must get its mode whatever the umask, and
+// check must pass on the result. After one file's mode is changed, a
+// second fetch must download that file again, and only it, to restore its
+// mode.
+func TestFetchSetsRecordedMode(t *testing.T) {
+	t.Parallel()
+
+	files := map[string][]byte{
+		testFileTxt: []byte("a file"),
+		"tool.sh":   []byte("#!/bin/sh\n"),
+	}
+	modes := map[string]os.FileMode{testFileTxt: 0o640, "tool.sh": 0o755}
+
+	sourceFs := afero.NewMemMapFs()
+	for p, content := range files {
+		require.NoError(t, afero.WriteFile(sourceFs, "/"+p, content, modes[p]))
+	}
+
+	scanner := mfer.NewScannerWithOptions(&mfer.ScannerOptions{
+		Fs:                 sourceFs,
+		IncludePermissions: true,
+	})
+	require.NoError(t, scanner.EnumerateFS(sourceFs, "/", nil))
+
+	var manifest bytes.Buffer
+	require.NoError(t, scanner.ToManifest(context.Background(), &manifest, nil))
+
+	tree := fetchTestHandler(manifest.Bytes(), files)
+
+	var (
+		mu        sync.Mutex
+		requested []string
+	)
+
+	server := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			mu.Lock()
+
+			requested = append(requested, r.URL.Path)
+
+			mu.Unlock()
+
+			tree.ServeHTTP(w, r)
+		}))
+	defer server.Close()
+
+	dest := t.TempDir()
+	fetch := []string{testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL}
+
+	opts := testOpts(fetch, afero.NewOsFs())
+	require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+
+	for p, mode := range modes {
+		info, err := os.Stat(filepath.Join(dest, p))
+		require.NoError(t, err)
+		assert.Equal(t, mode, info.Mode().Perm(), p)
+	}
+
+	opts = testOpts([]string{
+		testApp, cmdCheck, "-q", testFlagBase, dest,
+		filepath.Join(dest, defaultManifestName),
+	}, afero.NewOsFs())
+	require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+
+	require.NoError(t, os.Chmod(filepath.Join(dest, testFileTxt), 0o600))
+
+	mu.Lock()
+	requested = nil
+	mu.Unlock()
+
+	opts = testOpts(fetch, afero.NewOsFs())
+	require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+
+	info, err := os.Stat(filepath.Join(dest, testFileTxt))
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0o640), info.Mode().Perm())
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	assert.ElementsMatch(t,
+		[]string{"/" + defaultManifestName, "/" + testFileTxt}, requested)
+}
+
+// TestFetchRefusesModeOutsidePermissionBits fetches manifests that record
+// a mode with the setuid bit, once as Unix writes it (04755) and once as
+// Go keeps it in os.FileMode. fetch must refuse both before it creates the
+// destination or requests any file.
+func TestFetchRefusesModeOutsidePermissionBits(t *testing.T) {
+	t.Parallel()
+
+	files := map[string][]byte{testFileTxt: []byte("a file")}
+
+	assertFetchRefused(t, manifestWithMode(t, testFileTxt, files[testFileTxt], 0o4755),
+		files, "manifest lists a mode outside 0777: file.txt (04755)")
+
+	assertFetchRefused(t,
+		manifestWithMode(t, testFileTxt, files[testFileTxt],
+			uint32(os.ModeSetuid|0o755)),
+		files, "manifest lists a mode outside 0777: file.txt (040000755)")
+}
+
+// TestDownloadFileSetsOnlyPermissionBits downloads a file whose entry
+// records mode 0755 together with setuid, setgid or sticky, as Go keeps
+// them in os.FileMode. fetchManifest would refuse such an entry; called
+// directly, downloadFile must still set only the permission bits.
+func TestDownloadFileSetsOnlyPermissionBits(t *testing.T) {
+	t.Parallel()
+
+	content := []byte("#!/bin/sh\n")
+
+	digest := sha256.Sum256(content)
+	hash, err := multihash.Encode(digest[:], multihash.SHA2_256)
+	require.NoError(t, err)
+
+	server := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+			_, _ = w.Write(content)
+		}))
+	defer server.Close()
+
+	for _, special := range []os.FileMode{os.ModeSetuid, os.ModeSetgid, os.ModeSticky} {
+		entry := &mfer.MFFilePath{
+			Path:   testFileTxt,
+			Size:   int64(len(content)),
+			Hashes: []*mfer.MFFileChecksum{{MultiHash: hash}},
+			Mode:   uint32(special | 0o755),
+		}
+
+		dest := t.TempDir()
+
+		err := downloadFile(context.Background(), testClient(),
+			server.URL+"/"+testFileTxt, dest, testFileTxt, entry, nil)
+		require.NoError(t, err, special)
+
+		info, err := os.Stat(filepath.Join(dest, testFileTxt))
+		require.NoError(t, err)
+		assert.Equal(t, os.FileMode(0o755), info.Mode(), special)
+	}
+}
+
+// manifestWithMode returns a manifest listing one file, path, with content
+// and the given recorded mode. It is assembled by hand, since the builder
+// never records a mode outside 0777.
+func manifestWithMode(t *testing.T, path string, content []byte, mode uint32) []byte {
+	t.Helper()
+
+	digest := sha256.Sum256(content)
+	hash, err := multihash.Encode(digest[:], multihash.SHA2_256)
+	require.NoError(t, err)
+
+	id := uuid.New()
+
+	inner, err := proto.Marshal(&mfer.MFFile{
+		Version: mfer.MFFile_VERSION_ONE,
+		Files: []*mfer.MFFilePath{{
+			Path:   path,
+			Size:   int64(len(content)),
+			Hashes: []*mfer.MFFileChecksum{{MultiHash: hash}},
+			Mode:   mode,
+		}},
+		Uuid: id[:],
+	})
+	require.NoError(t, err)
+
+	encoder, err := zstd.NewWriter(nil)
+	require.NoError(t, err)
+
+	compressed := encoder.EncodeAll(inner, nil)
+	require.NoError(t, encoder.Close())
+
+	sum := sha256.Sum256(compressed)
+
+	outer, err := proto.Marshal(&mfer.MFFileOuter{
+		Version:         mfer.MFFileOuter_VERSION_ONE,
+		CompressionType: mfer.MFFileOuter_COMPRESSION_ZSTD,
+		Size:            int64(len(inner)),
+		Sha256:          sum[:],
+		Uuid:            id[:],
+		InnerMessage:    compressed,
+	})
+	require.NoError(t, err)
+
+	return append([]byte(mfer.MAGIC), outer...)
+}
+
 // builtManifest returns a manifest of files, built directly rather than
 // scanned, since a scan lists no hidden files and never a path starting
 // with "./".
@@ -1309,7 +1501,7 @@ func builtManifest(t *testing.T, files map[string][]byte) []byte {
 
 	for p, content := range files {
 		_, err := builder.AddFile(mfer.RelFilePath(p), mfer.FileSize(len(content)),
-			mfer.ModTime(time.Now()), bytes.NewReader(content), nil)
+			mfer.ModTime(time.Now()), 0, bytes.NewReader(content), nil)
 		require.NoError(t, err)
 	}