fetch: destination directory, skip files already present, save the manifest, require a signer (closes #101)

fetch takes --dest (default .) and writes every file there through the
existing symlink and hard-link guards, which now work relative to that
directory. A file already there with the listed size and hash is
skipped; a leftover temp file is still replaced. Once every file
verifies, the manifest is saved as index.mf through the same temp file
and rename, so check runs on the result; a manifest that lists index.mf
or its temp name at the top of the tree is refused, since saving would
replace that file. --require-signature is shared with check and
enforced through verifyRequiredSigner. Both refusals come before any
file is downloaded or anything is written.

Model: opus-5-5

internal/cli/fetch.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/multiformats/go-multihash"
+	"github.com/spf13/afero"
 	"github.com/urfave/cli/v2"
 	"sneak.berlin/go/mfer/internal/log"
 	"sneak.berlin/go/mfer/mfer"
@@ -90,6 +91,10 @@ var (
 	// errHashMismatch indicates a downloaded file whose hash matches no
 	// manifest hash.
 	errHashMismatch = errors.New("hash mismatch")
+	// errManifestNameListed indicates a manifest that lists a file where
+	// fetch saves the manifest.
+	errManifestNameListed = errors.New(
+		"manifest lists a file where fetch saves the manifest")
 )
 
 // DownloadProgress reports the progress of a single file download.
@@ -239,20 +244,34 @@ func manifestBaseURL(manifestURL string) (*url.URL, error) {
 	return parsed.JoinPath(".."), nil
 }
 
-// downloadManifestFiles downloads every file in the manifest, reporting
-// progress on the progress channel.
+// downloadManifestFiles downloads every file in the manifest into dest,
+// reporting progress on the progress channel. A file already present in
+// dest is skipped. It returns how many files it downloaded and their
+// total size.
 func downloadManifestFiles(
 	ctx context.Context,
 	client retryingClient,
 	baseURL *url.URL,
+	dest string,
 	files []*mfer.MFFilePath,
 	progress chan<- DownloadProgress,
-) error {
+) (int, int64, error) {
+	var (
+		downloaded      int
+		downloadedBytes int64
+	)
+
 	for _, f := range files {
 		// Sanitize the path to prevent path traversal attacks
 		localPath, err := sanitizePath(f.GetPath())
 		if err != nil {
-			return fmt.Errorf("invalid path in manifest: %w", err)
+			return 0, 0, fmt.Errorf("invalid path in manifest: %w", err)
+		}
+
+		if alreadyPresent(dest, localPath, f) {
+			log.Infof("skipping %s: already present", f.GetPath())
+
+			continue
 		}
 
 		// JoinPath takes escaped path text, so a name such as "100%.txt"
@@ -260,13 +279,53 @@ func downloadManifestFiles(
 		fileURL := baseURL.JoinPath(encodeFilePath(f.GetPath())).String()
 		log.Infof("fetching %s", f.GetPath())
 
-		err = downloadFile(ctx, client, fileURL, localPath, f, progress)
+		err = downloadFile(ctx, client, fileURL, dest, localPath, f, progress)
 		if err != nil {
-			return fmt.Errorf("failed to download %s: %w", f.GetPath(), err)
+			return 0, 0, fmt.Errorf("failed to download %s: %w", f.GetPath(), err)
 		}
+
+		downloaded++
+		downloadedBytes += f.GetSize()
 	}
 
-	return nil
+	return downloaded, downloadedBytes, nil
+}
+
+// alreadyPresent reports whether localPath under dest is a regular file
+// with the size and one of the hashes the manifest lists for entry. It
+// hashes the whole file, since a matching size alone would accept a
+// corrupted or partly written one. A file it cannot read, or reaches only
+// through a symlink, is not present: fetch downloads it, and the download
+// reports the problem.
+func alreadyPresent(dest, localPath string, entry *mfer.MFFilePath) bool {
+	if checkNoSymlinks(dest, localPath) != nil {
+		return false
+	}
+
+	path := filepath.Join(dest, localPath)
+
+	info, err := os.Lstat(path)
+	if err != nil || !info.Mode().IsRegular() || info.Size() != entry.GetSize() {
+		return false
+	}
+
+	// G304: localPath is a relative path that sanitizePath keeps inside
+	// dest as text, and checkNoSymlinks just found no symlink in it.
+	f, err := os.Open(path) //nolint:gosec // G304: see comment above
+	if err != nil {
+		return false
+	}
+
+	defer func() { _ = f.Close() }()
+
+	h := sha256.New()
+
+	_, err = io.Copy(h, f)
+	if err != nil {
+		return false
+	}
+
+	return verifyDownloadedHash(h.Sum(nil), entry) == nil
 }
 
 func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
@@ -291,43 +350,22 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
 		firstDelay: firstRetryDelay,
 	}
 
-	log.Infof("fetching manifest from %s", manifestURL)
-
-	// Read the whole manifest before parsing it, so that a connection
-	// lost partway through is retried rather than reported as a bad
-	// manifest.
-	var manifestData []byte
-
-	err = client.get(ctx.Context, manifestURL, func(resp *http.Response) error {
-		var readErr error
-
-		manifestData, readErr = io.ReadAll(resp.Body)
-
-		return readErr
-	})
+	manifestData, files, err := fetchManifest(ctx, client, manifestURL)
 	if err != nil {
-		return fmt.Errorf("failed to fetch manifest: %w", err)
+		return err
 	}
 
-	// Parse manifest
-	manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
-	if err != nil {
-		return fmt.Errorf("failed to parse manifest: %w", err)
-	}
-
-	files := manifest.Files()
-	log.Infof("manifest contains %d files", len(files))
-
 	// Compute base URL (directory containing manifest)
 	baseURL, err := manifestBaseURL(manifestURL)
 	if err != nil {
 		return err
 	}
 
-	// Calculate total bytes to download
-	var totalBytes int64
-	for _, f := range files {
-		totalBytes += f.GetSize()
+	dest := ctx.String(flagDest)
+
+	err = os.MkdirAll(dest, dirPerms)
+	if err != nil {
+		return fmt.Errorf("failed to create destination directory %s: %w", dest, err)
 	}
 
 	// Create progress channel and start progress reporter goroutine
@@ -340,7 +378,8 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
 	startTime := time.Now()
 
 	// Download each file
-	dlErr := downloadManifestFiles(ctx.Context, client, baseURL, files, progress)
+	downloaded, downloadedBytes, dlErr := downloadManifestFiles(
+		ctx.Context, client, baseURL, dest, files, progress)
 
 	close(progress)
 	<-done
@@ -349,15 +388,150 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
 		return dlErr
 	}
 
+	// Saved only now that every file is in place and verified, so that
+	// "mfer check" can verify the tree later.
+	err = saveManifest(dest, manifestData)
+	if err != nil {
+		return fmt.Errorf("failed to save manifest: %w", err)
+	}
+
 	// Print summary
 	elapsed := time.Since(startTime)
-	avgBytesPerSec := float64(totalBytes) / elapsed.Seconds()
+	avgBytesPerSec := float64(downloadedBytes) / elapsed.Seconds()
 	avgRate := formatBitrate(avgBytesPerSec * bitsPerByte)
-	log.Infof("downloaded %d files (%s) in %.1fs (%s avg)",
-		len(files),
-		humanize.IBytes(safeUint64(totalBytes)),
+	log.Infof("downloaded %d files (%s) in %.1fs (%s avg), skipped %d already present",
+		downloaded,
+		humanize.IBytes(safeUint64(downloadedBytes)),
 		elapsed.Seconds(),
-		avgRate)
+		avgRate,
+		len(files)-downloaded)
+	log.Infof("saved manifest to %s", filepath.Join(dest, defaultManifestName))
+
+	return nil
+}
+
+// fetchManifest downloads the manifest at manifestURL and parses it,
+// enforcing --require-signature if it is given and refusing a manifest
+// that lists a file where it will be saved. It returns the manifest as
+// downloaded, to be saved once the files are in place, and the files it
+// lists.
+func fetchManifest(
+	ctx *cli.Context, client retryingClient, manifestURL string,
+) ([]byte, []*mfer.MFFilePath, error) {
+	log.Infof("fetching manifest from %s", manifestURL)
+
+	// Read the whole manifest before parsing it, so that a connection
+	// lost partway through is retried rather than reported as a bad
+	// manifest.
+	var manifestData []byte
+
+	err := client.get(ctx.Context, manifestURL, func(resp *http.Response) error {
+		var readErr error
+
+		manifestData, readErr = io.ReadAll(resp.Body)
+
+		return readErr
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err)
+	}
+
+	// Parse manifest
+	manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to parse manifest: %w", err)
+	}
+
+	requiredSigner := ctx.String(flagRequireSignature)
+	if requiredSigner != "" {
+		err = verifyFetchedSigner(ctx, manifestData, requiredSigner)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	files := manifest.Files()
+
+	err = checkManifestNameUnlisted(files)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	log.Infof("manifest contains %d files", len(files))
+
+	return manifestData, files, nil
+}
+
+// checkManifestNameUnlisted returns an error if files lists a file or
+// directory at the top of the tree under the name fetch saves the
+// manifest as, or under that name's temp file. Saving the manifest would
+// replace or remove it, or fail once every file was downloaded, leaving a
+// tree check rejects. Names are compared ignoring case, since on a
+// case-insensitive filesystem INDEX.MF and index.mf are one file.
+func checkManifestNameUnlisted(files []*mfer.MFFilePath) error {
+	for _, f := range files {
+		top, _, _ := strings.Cut(filepath.Clean(f.GetPath()), string(filepath.Separator))
+
+		if strings.EqualFold(top, defaultManifestName) ||
+			strings.EqualFold(top, tempPathFor(defaultManifestName)) {
+			return fmt.Errorf("%w: %s", errManifestNameListed, f.GetPath())
+		}
+	}
+
+	return nil
+}
+
+// verifyFetchedSigner enforces --require-signature on the fetched manifest
+// exactly as check does. verifyRequiredSigner takes a Checker, which loads
+// its manifest from a file, so the manifest is handed to it as a file in
+// memory.
+func verifyFetchedSigner(
+	ctx *cli.Context, manifestData []byte, requiredSigner string,
+) error {
+	memFs := afero.NewMemMapFs()
+	manifestPath := "/" + defaultManifestName
+
+	err := afero.WriteFile(memFs, manifestPath, manifestData, filePerms)
+	if err != nil {
+		return err
+	}
+
+	chk, err := mfer.NewChecker(&mfer.CheckerOptions{
+		ManifestPath: manifestPath,
+		BasePath:     "/",
+		Fs:           memFs,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to load manifest: %w", err)
+	}
+
+	return verifyRequiredSigner(ctx.Context, chk, requiredSigner)
+}
+
+// saveManifest writes the fetched manifest into dest under the default
+// manifest name, the way fetch writes every file: to a new temp file that
+// is then renamed into place.
+func saveManifest(dest string, manifestData []byte) error {
+	tmpPath := tempPathFor(defaultManifestName)
+
+	out, err := createTempFile(dest, tmpPath)
+	if err != nil {
+		return err
+	}
+
+	_, writeErr := out.Write(manifestData)
+	closeErr := out.Close()
+
+	err = errors.Join(writeErr, closeErr)
+	if err == nil {
+		err = moveIntoPlace(dest, tmpPath, defaultManifestName)
+	}
+
+	if err != nil {
+		_ = os.Remove(filepath.Join(dest, tmpPath))
+
+		return err
+	}
 
 	return nil
 }
@@ -401,14 +575,15 @@ func sanitizePath(p string) (string, error) {
 	return cleaned, nil
 }
 
-// checkNoSymlinks returns an error if any part of the relative path p
-// already exists as a symlink. sanitizePath checks p only as text, so
-// without this a symlink inside the target directory could send a write
-// to p outside of it. Parts that do not exist yet are fine: fetch creates
-// them as plain directories and files. Call it immediately before each
-// write: a symlink created after it returns is not caught.
-func checkNoSymlinks(p string) error {
-	current := ""
+// checkNoSymlinks returns an error if any part of p, a path relative to
+// dest, already exists under dest as a symlink. dest itself is the user's
+// choice and may be one. sanitizePath checks p only as text, so without
+// this a symlink inside dest could send a write to p outside of it. Parts
+// that do not exist yet are fine: fetch creates them as plain directories
+// and files. Call it immediately before each write: a symlink created
+// after it returns is not caught.
+func checkNoSymlinks(dest, p string) error {
+	current := dest
 
 	for _, part := range strings.Split(p, string(filepath.Separator)) {
 		current = filepath.Join(current, part)
@@ -548,13 +723,14 @@ func verifyDownloadedHash(digest []byte, entry *mfer.MFFilePath) error {
 	return errHashMismatch
 }
 
-// downloadFile downloads a URL to a local file path with hash verification.
-// It downloads to a temporary file, verifies the hash, then renames to the final path.
-// Progress is reported via the progress channel.
+// downloadFile downloads a URL to localPath, a path relative to dest, with
+// hash verification. It downloads to a temporary file, verifies the hash,
+// then renames to the final path. Progress is reported via the progress
+// channel.
 func downloadFile(
 	ctx context.Context,
 	client retryingClient,
-	fileURL, localPath string,
+	fileURL, dest, localPath string,
 	entry *mfer.MFFilePath,
 	progress chan<- DownloadProgress,
 ) error {
@@ -568,11 +744,13 @@ func downloadFile(
 	// Create parent directories if needed
 	dir := filepath.Dir(localPath)
 	if dir != "" && dir != "." {
-		err = checkNoSymlinks(dir)
+		err = checkNoSymlinks(dest, dir)
 		if err != nil {
 			return err
 		}
 
+		dir = filepath.Join(dest, dir)
+
 		err = os.MkdirAll(dir, dirPerms)
 		if err != nil {
 			return fmt.Errorf("failed to create directory %s: %w", dir, err)
@@ -582,17 +760,61 @@ func downloadFile(
 	tmpPath := tempPathFor(localPath)
 
 	return client.get(ctx, fileURL, func(resp *http.Response) error {
-		return saveResponse(resp, tmpPath, localPath, entry, progress)
+		return saveResponse(resp, dest, tmpPath, localPath, entry, progress)
 	})
 }
 
+// createTempFile creates tmpPath, a path relative to dest, as a new empty
+// file.
+func createTempFile(dest, tmpPath string) (*os.File, error) {
+	err := checkNoSymlinks(dest, tmpPath)
+	if err != nil {
+		return nil, err
+	}
+
+	path := filepath.Join(dest, tmpPath)
+
+	// Remove whatever is at tmpPath, such as a leftover from an
+	// interrupted run, rather than write into it: it may be a hard link
+	// to a file outside dest, and removing a hard link removes only this
+	// name. If the removal fails, O_EXCL below makes the create fail.
+	_ = os.Remove(path)
+
+	// Create the temp file only if nothing is at tmpPath (O_EXCL).
+	//
+	// G304: tmpPath is a relative path that sanitizePath keeps inside dest
+	// as text, and checkNoSymlinks just found no symlink in it.
+	out, err := os.OpenFile( //nolint:gosec // G304: see comment above
+		path, os.O_RDWR|os.O_CREATE|os.O_EXCL, filePerms)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temp file: %w", err)
+	}
+
+	return out, nil
+}
+
+// moveIntoPlace renames tmpPath to localPath, both relative to dest.
+func moveIntoPlace(dest, tmpPath, localPath string) error {
+	err := checkNoSymlinks(dest, localPath)
+	if err != nil {
+		return err
+	}
+
+	err = os.Rename(filepath.Join(dest, tmpPath), filepath.Join(dest, localPath))
+	if err != nil {
+		return fmt.Errorf("failed to rename temp file: %w", err)
+	}
+
+	return nil
+}
+
 // saveResponse writes resp's body to tmpPath, verifies it against entry,
-// and renames it to localPath. It starts a new temp file each time and
-// removes it on failure, so a retry after a failed try never appends to
-// or keeps a partial file.
+// and renames it to localPath, both paths relative to dest. It starts a
+// new temp file each time and removes it on failure, so a retry after a
+// failed try never appends to or keeps a partial file.
 func saveResponse(
 	resp *http.Response,
-	tmpPath, localPath string,
+	dest, tmpPath, localPath string,
 	entry *mfer.MFFilePath,
 	progress chan<- DownloadProgress,
 ) error {
@@ -604,29 +826,11 @@ func saveResponse(
 		totalBytes = expectedSize
 	}
 
-	err := checkNoSymlinks(tmpPath)
+	out, err := createTempFile(dest, tmpPath)
 	if err != nil {
 		return err
 	}
 
-	// Remove whatever is at tmpPath, such as a leftover from an
-	// interrupted run, rather than write into it: it may be a hard link
-	// to a file outside the target directory, and removing a hard link
-	// removes only this name. If the removal fails, O_EXCL below makes
-	// the create fail.
-	_ = os.Remove(tmpPath)
-
-	// Create the temp file only if nothing is at tmpPath (O_EXCL).
-	//
-	// G304: tmpPath is a relative path that sanitizePath keeps inside the
-	// target directory as text, and checkNoSymlinks just found no symlink
-	// in it.
-	out, err := os.OpenFile( //nolint:gosec // G304: see comment above
-		tmpPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, filePerms)
-	if err != nil {
-		return fmt.Errorf("failed to create temp file: %w", err)
-	}
-
 	// Set up hash computation
 	h := sha256.New()
 
@@ -646,10 +850,10 @@ func saveResponse(
 	closeErr := out.Close()
 
 	err = finishDownload(
-		tmpPath, localPath, written, expectedSize, h.Sum(nil), entry,
+		dest, tmpPath, localPath, written, expectedSize, h.Sum(nil), entry,
 		copyErr, closeErr)
 	if err != nil {
-		_ = os.Remove(tmpPath)
+		_ = os.Remove(filepath.Join(dest, tmpPath))
 
 		return err
 	}
@@ -660,7 +864,7 @@ func saveResponse(
 // finishDownload validates the copy result, verifies size and hash, and
 // moves the temp file into place. On error the caller removes tmpPath.
 func finishDownload(
-	tmpPath, localPath string,
+	dest, tmpPath, localPath string,
 	written, expectedSize int64,
 	digest []byte,
 	entry *mfer.MFFilePath,
@@ -686,16 +890,5 @@ func finishDownload(
 		return err
 	}
 
-	err = checkNoSymlinks(localPath)
-	if err != nil {
-		return err
-	}
-
-	// Rename temp file to final path
-	err = os.Rename(tmpPath, localPath)
-	if err != nil {
-		return fmt.Errorf("failed to rename temp file: %w", err)
-	}
-
-	return nil
+	return moveIntoPlace(dest, tmpPath, localPath)
 }