mfer/internal/cli/fetch.go

package cli

import (
	"bytes"
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/multiformats/go-multihash"
	"github.com/urfave/cli/v2"
	"sneak.berlin/go/mfer/internal/log"
	"sneak.berlin/go/mfer/mfer"
)

const (
	// progressChanBuffer is the buffer size of the download progress
	// channel.
	progressChanBuffer = 10

	// bitsPerByte converts a bytes-per-second rate to bits per second.
	bitsPerByte = 8

	// dirPerms is the permission mode for directories created for
	// downloaded files. Fetched trees are content that is normally
	// published (served by a web server, read by another uid), so the
	// traversal bit for group and other must stay set.
	dirPerms os.FileMode = 0o755

	// filePerms is the permission mode, before the umask, for downloaded
	// files. It is the mode os.Create uses; like dirPerms, it keeps group
	// and other read access.
	filePerms os.FileMode = 0o666

	// Bitrate unit thresholds in bits per second.
	bpsPerGbps = 1e9
	bpsPerMbps = 1e6
	bpsPerKbps = 1e3
)

var (
	// errURLRequired indicates the fetch command was run without a URL
	// argument.
	errURLRequired = errors.New("URL argument required")
	// errEmptyPath indicates an empty file path in the manifest.
	errEmptyPath = errors.New("empty path")
	// errAbsolutePath indicates an absolute file path in the manifest.
	errAbsolutePath = errors.New("absolute path not allowed")
	// errPathTraversal indicates a manifest path escaping the target
	// directory.
	errPathTraversal = errors.New("path traversal not allowed")
	// errSymlinkInPath indicates a manifest path running through a
	// symlink that already exists in the target directory.
	errSymlinkInPath = errors.New("symlink in path not allowed")
	// errSizeMismatch indicates a downloaded file with an unexpected
	// size.
	errSizeMismatch = errors.New("size mismatch")
	// errHashMismatch indicates a downloaded file whose hash matches no
	// manifest hash.
	errHashMismatch = errors.New("hash mismatch")
)

// DownloadProgress reports the progress of a single file download.
type DownloadProgress struct {
	Path        string        // File path being downloaded
	BytesRead   int64         // Bytes downloaded so far
	TotalBytes  int64         // Total expected bytes (-1 if unknown)
	BytesPerSec float64       // Current download rate
	ETA         time.Duration // Estimated time to completion
}

// httpGet issues a GET request for the given URL using the provided
// context and returns the response. The caller must close the body.
//
// Errors are returned unwrapped: this helper replaced direct http.Get
// calls, and each caller already supplies its own context string, so
// adding one here would change user-visible messages.
func httpGet(ctx context.Context, fileURL string) (*http.Response, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
	if err != nil {
		return nil, err
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

// reportDownloadProgress renders download progress until the channel
// closes, then closes done.
func reportDownloadProgress(progress <-chan DownloadProgress, done chan<- struct{}) {
	defer close(done)

	for p := range progress {
		rate := formatBitrate(p.BytesPerSec * bitsPerByte)
		if p.ETA > 0 {
			log.Infof("%s: %s/%s, %s, ETA %s",
				p.Path, humanize.IBytes(safeUint64(p.BytesRead)),
				humanize.IBytes(safeUint64(p.TotalBytes)),
				rate, p.ETA.Round(time.Second))
		} else {
			log.Infof("%s: %s/%s, %s",
				p.Path, humanize.IBytes(safeUint64(p.BytesRead)),
				humanize.IBytes(safeUint64(p.TotalBytes)), rate)
		}
	}
}

// manifestBaseURL returns the URL of the directory containing the
// manifest, with a trailing slash.
func manifestBaseURL(manifestURL string) (*url.URL, error) {
	baseURL, err := url.Parse(manifestURL)
	if err != nil {
		return nil, fmt.Errorf("fetch: invalid manifest URL: %w", err)
	}

	baseURL.Path = path.Dir(baseURL.Path)
	if !strings.HasSuffix(baseURL.Path, "/") {
		baseURL.Path += "/"
	}

	return baseURL, nil
}

// downloadManifestFiles downloads every file in the manifest, reporting
// progress on the progress channel.
func downloadManifestFiles(
	ctx context.Context,
	baseURL *url.URL,
	files []*mfer.MFFilePath,
	progress chan<- DownloadProgress,
) error {
	for _, f := range files {
		// Sanitize the path to prevent path traversal attacks
		localPath, err := sanitizePath(f.GetPath())
		if err != nil {
			return fmt.Errorf("invalid path in manifest: %w", err)
		}

		fileURL := baseURL.String() + encodeFilePath(f.GetPath())
		log.Infof("fetching %s", f.GetPath())

		err = downloadFile(ctx, fileURL, localPath, f, progress)
		if err != nil {
			return fmt.Errorf("failed to download %s: %w", f.GetPath(), err)
		}
	}

	return nil
}

func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
	log.Debug("fetchManifestOperation()")

	if ctx.Args().Len() == 0 {
		return errURLRequired
	}

	inputURL := ctx.Args().Get(0)

	manifestURL, err := resolveManifestURL(inputURL)
	if err != nil {
		return fmt.Errorf("invalid URL: %w", err)
	}

	log.Infof("fetching manifest from %s", manifestURL)

	// Fetch manifest
	resp, err := httpGet(ctx.Context, manifestURL)
	if err != nil {
		return fmt.Errorf("failed to fetch manifest: %w", err)
	}

	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("failed to fetch manifest: %w %d",
			errHTTPStatus, resp.StatusCode)
	}

	// Parse manifest
	manifest, err := mfer.NewManifestFromReader(resp.Body)
	if err != nil {
		return fmt.Errorf("failed to parse manifest: %w", err)
	}

	files := manifest.Files()
	log.Infof("manifest contains %d files", len(files))

	// Compute base URL (directory containing manifest)
	baseURL, err := manifestBaseURL(manifestURL)
	if err != nil {
		return err
	}

	// Calculate total bytes to download
	var totalBytes int64
	for _, f := range files {
		totalBytes += f.GetSize()
	}

	// Create progress channel and start progress reporter goroutine
	progress := make(chan DownloadProgress, progressChanBuffer)
	done := make(chan struct{})

	go reportDownloadProgress(progress, done)

	// Track download start time
	startTime := time.Now()

	// Download each file
	dlErr := downloadManifestFiles(ctx.Context, baseURL, files, progress)

	close(progress)
	<-done

	if dlErr != nil {
		return dlErr
	}

	// Print summary
	elapsed := time.Since(startTime)
	avgBytesPerSec := float64(totalBytes) / elapsed.Seconds()
	avgRate := formatBitrate(avgBytesPerSec * bitsPerByte)
	log.Infof("downloaded %d files (%s) in %.1fs (%s avg)",
		len(files),
		humanize.IBytes(safeUint64(totalBytes)),
		elapsed.Seconds(),
		avgRate)

	return nil
}

// encodeFilePath URL-encodes each segment of a file path while preserving slashes.
func encodeFilePath(p string) string {
	segments := strings.Split(p, "/")
	for i, seg := range segments {
		segments[i] = url.PathEscape(seg)
	}

	return strings.Join(segments, "/")
}

// sanitizePath validates and sanitizes a file path from the manifest.
// It prevents path traversal attacks and rejects unsafe paths.
func sanitizePath(p string) (string, error) {
	// Reject empty paths
	if p == "" {
		return "", errEmptyPath
	}

	// Reject absolute paths
	if filepath.IsAbs(p) {
		return "", fmt.Errorf("%w: %s", errAbsolutePath, p)
	}

	// Clean the path to resolve . and ..
	cleaned := filepath.Clean(p)

	// Reject paths that escape the current directory
	if strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) || cleaned == ".." {
		return "", fmt.Errorf("%w: %s", errPathTraversal, p)
	}

	// Also check for absolute paths after cleaning (handles edge cases)
	if filepath.IsAbs(cleaned) {
		return "", fmt.Errorf("%w: %s", errAbsolutePath, p)
	}

	return cleaned, nil
}

// checkNoSymlinks returns an error if any part of the relative path p
// already exists as a symlink. sanitizePath checks p only as text, so
// without this a symlink inside the target directory could send a write
// to p outside of it. Parts that do not exist yet are fine: fetch creates
// them as plain directories and files. Call it immediately before each
// write: a symlink created after it returns is not caught.
func checkNoSymlinks(p string) error {
	current := ""

	for _, part := range strings.Split(p, string(filepath.Separator)) {
		current = filepath.Join(current, part)

		info, err := os.Lstat(current)
		if errors.Is(err, os.ErrNotExist) {
			return nil
		}

		if err != nil {
			return fmt.Errorf("failed to check %s for a symlink: %w", current, err)
		}

		if info.Mode()&os.ModeSymlink != 0 {
			return fmt.Errorf("%w: %s", errSymlinkInPath, current)
		}
	}

	return nil
}

// resolveManifestURL takes a URL and returns the manifest URL.
// If the URL already ends with .mf, it's returned as-is.
// Otherwise, index.mf is appended.
func resolveManifestURL(inputURL string) (string, error) {
	parsed, err := url.Parse(inputURL)
	if err != nil {
		return "", err
	}

	// Check if URL already ends with .mf
	if strings.HasSuffix(parsed.Path, ".mf") {
		return inputURL, nil
	}

	// Ensure path ends with /
	if !strings.HasSuffix(parsed.Path, "/") {
		parsed.Path += "/"
	}

	// Append index.mf
	parsed.Path += "index.mf"

	return parsed.String(), nil
}

// progressWriter wraps an io.Writer and reports progress to a channel.
type progressWriter struct {
	w         io.Writer
	path      string
	total     int64
	written   int64
	startTime time.Time
	progress  chan<- DownloadProgress
}

func (pw *progressWriter) Write(p []byte) (int, error) {
	n, err := pw.w.Write(p)

	pw.written += int64(n)
	if pw.progress != nil {
		var (
			bytesPerSec float64
			eta         time.Duration
		)

		elapsed := time.Since(pw.startTime)
		if elapsed > 0 && pw.written > 0 {
			bytesPerSec = float64(pw.written) / elapsed.Seconds()
			if bytesPerSec > 0 && pw.total > 0 {
				remainingBytes := pw.total - pw.written
				eta = time.Duration(float64(remainingBytes)/bytesPerSec) * time.Second
			}
		}

		sendProgress(pw.progress, DownloadProgress{
			Path:        pw.path,
			BytesRead:   pw.written,
			TotalBytes:  pw.total,
			BytesPerSec: bytesPerSec,
			ETA:         eta,
		})
	}

	return n, err
}

// formatBitrate formats a bits-per-second value with appropriate unit prefix.
func formatBitrate(bps float64) string {
	switch {
	case bps >= bpsPerGbps:
		return fmt.Sprintf("%.1f Gbps", bps/bpsPerGbps)
	case bps >= bpsPerMbps:
		return fmt.Sprintf("%.1f Mbps", bps/bpsPerMbps)
	case bps >= bpsPerKbps:
		return fmt.Sprintf("%.1f Kbps", bps/bpsPerKbps)
	default:
		return fmt.Sprintf("%.0f bps", bps)
	}
}

// sendProgress sends a progress update without blocking.
func sendProgress(ch chan<- DownloadProgress, p DownloadProgress) {
	select {
	case ch <- p:
	default:
	}
}

// tempPathFor computes the temporary download path for a local file.
// For dotfiles, just append .tmp (they're already hidden); for regular
// files, prefix with . and append .tmp.
func tempPathFor(localPath string) string {
	dir := filepath.Dir(localPath)
	base := filepath.Base(localPath)

	var tmpName string
	if strings.HasPrefix(base, ".") {
		tmpName = base + ".tmp"
	} else {
		tmpName = "." + base + ".tmp"
	}

	if dir == "" || dir == "." {
		return tmpName
	}

	return filepath.Join(dir, tmpName)
}

// verifyDownloadedHash checks the computed sha256 digest against the
// manifest entry's hashes; at least one must match.
func verifyDownloadedHash(digest []byte, entry *mfer.MFFilePath) error {
	computed, err := multihash.Encode(digest, multihash.SHA2_256)
	if err != nil {
		return fmt.Errorf("failed to encode hash: %w", err)
	}

	for _, hash := range entry.GetHashes() {
		if bytes.Equal(computed, hash.GetMultiHash()) {
			return nil
		}
	}

	return errHashMismatch
}

// downloadFile downloads a URL to a local file path with hash verification.
// It downloads to a temporary file, verifies the hash, then renames to the final path.
// Progress is reported via the progress channel.
func downloadFile(
	ctx context.Context,
	fileURL, localPath string,
	entry *mfer.MFFilePath,
	progress chan<- DownloadProgress,
) error {
	// Enforce the path invariant here rather than relying on the caller,
	// so every entry point to downloadFile gets the same treatment.
	localPath, err := sanitizePath(localPath)
	if err != nil {
		return fmt.Errorf("invalid path: %w", err)
	}

	// Create parent directories if needed
	dir := filepath.Dir(localPath)
	if dir != "" && dir != "." {
		err = checkNoSymlinks(dir)
		if err != nil {
			return err
		}

		err = os.MkdirAll(dir, dirPerms)
		if err != nil {
			return fmt.Errorf("failed to create directory %s: %w", dir, err)
		}
	}

	tmpPath := tempPathFor(localPath)

	// Fetch file
	resp, err := httpGet(ctx, fileURL)
	if err != nil {
		return fmt.Errorf("HTTP request failed: %w", err)
	}

	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("%w %d", errHTTPStatus, resp.StatusCode)
	}

	// Determine expected size
	expectedSize := entry.GetSize()

	totalBytes := resp.ContentLength
	if totalBytes < 0 {
		totalBytes = expectedSize
	}

	err = checkNoSymlinks(tmpPath)
	if err != nil {
		return err
	}

	// Remove whatever is at tmpPath, such as a leftover from an
	// interrupted run, rather than write into it: it may be a hard link
	// to a file outside the target directory, and removing a hard link
	// removes only this name. If the removal fails, O_EXCL below makes
	// the create fail.
	_ = os.Remove(tmpPath)

	// Create the temp file only if nothing is at tmpPath (O_EXCL).
	//
	// G304: tmpPath is a relative path that sanitizePath keeps inside the
	// target directory as text, and checkNoSymlinks just found no symlink
	// in it.
	out, err := os.OpenFile( //nolint:gosec // G304: see comment above
		tmpPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, filePerms)
	if err != nil {
		return fmt.Errorf("failed to create temp file: %w", err)
	}

	// Set up hash computation
	h := sha256.New()

	// Create progress-reporting writer that also computes hash
	pw := &progressWriter{
		w:         io.MultiWriter(out, h),
		path:      localPath,
		total:     totalBytes,
		startTime: time.Now(),
		progress:  progress,
	}

	// Copy content while hashing and reporting progress
	written, copyErr := io.Copy(pw, resp.Body)

	// Close file before checking errors (to flush writes)
	closeErr := out.Close()

	err = finishDownload(
		tmpPath, localPath, written, expectedSize, h.Sum(nil), entry,
		copyErr, closeErr)
	if err != nil {
		_ = os.Remove(tmpPath)

		return err
	}

	return nil
}

// finishDownload validates the copy result, verifies size and hash, and
// moves the temp file into place. On error the caller removes tmpPath.
func finishDownload(
	tmpPath, localPath string,
	written, expectedSize int64,
	digest []byte,
	entry *mfer.MFFilePath,
	copyErr, closeErr error,
) error {
	if copyErr != nil {
		return copyErr
	}

	if closeErr != nil {
		return closeErr
	}

	// Verify size
	if written != expectedSize {
		return fmt.Errorf("%w: expected %d bytes, got %d",
			errSizeMismatch, expectedSize, written)
	}

	// Verify hash against manifest (at least one must match)
	err := verifyDownloadedHash(digest, entry)
	if err != nil {
		return err
	}

	err = checkNoSymlinks(localPath)
	if err != nil {
		return err
	}

	// Rename temp file to final path
	err = os.Rename(tmpPath, localPath)
	if err != nil {
		return fmt.Errorf("failed to rename temp file: %w", err)
	}

	return nil
}