package cli import ( "bytes" "context" "crypto/sha256" "errors" "fmt" "io" "net/http" "net/url" "os" "path" "path/filepath" "strings" "time" "github.com/dustin/go-humanize" "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" "sneak.berlin/go/mfer/internal/log" "sneak.berlin/go/mfer/mfer" ) const ( // progressChanBuffer is the buffer size of the download progress // channel. progressChanBuffer = 10 // bitsPerByte converts a bytes-per-second rate to bits per second. bitsPerByte = 8 // dirPerms is the permission mode for directories created for // downloaded files. Fetched trees are content that is normally // published (served by a web server, read by another uid), so the // traversal bit for group and other must stay set. dirPerms os.FileMode = 0o755 // Bitrate unit thresholds in bits per second. bpsPerGbps = 1e9 bpsPerMbps = 1e6 bpsPerKbps = 1e3 ) var ( // errURLRequired indicates the fetch command was run without a URL // argument. errURLRequired = errors.New("URL argument required") // errEmptyPath indicates an empty file path in the manifest. errEmptyPath = errors.New("empty path") // errAbsolutePath indicates an absolute file path in the manifest. errAbsolutePath = errors.New("absolute path not allowed") // errPathTraversal indicates a manifest path escaping the target // directory. errPathTraversal = errors.New("path traversal not allowed") // errSizeMismatch indicates a downloaded file with an unexpected // size. errSizeMismatch = errors.New("size mismatch") // errHashMismatch indicates a downloaded file whose hash matches no // manifest hash. errHashMismatch = errors.New("hash mismatch") ) // DownloadProgress reports the progress of a single file download. type DownloadProgress struct { Path string // File path being downloaded BytesRead int64 // Bytes downloaded so far TotalBytes int64 // Total expected bytes (-1 if unknown) BytesPerSec float64 // Current download rate ETA time.Duration // Estimated time to completion } // httpGet issues a GET request for the given URL using the provided // context and returns the response. The caller must close the body. // // Errors are returned unwrapped: this helper replaced direct http.Get // calls, and each caller already supplies its own context string, so // adding one here would change user-visible messages. func httpGet(ctx context.Context, fileURL string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil) if err != nil { return nil, err } resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } return resp, nil } // reportDownloadProgress renders download progress until the channel // closes, then closes done. func reportDownloadProgress(progress <-chan DownloadProgress, done chan<- struct{}) { defer close(done) for p := range progress { rate := formatBitrate(p.BytesPerSec * bitsPerByte) if p.ETA > 0 { log.Infof("%s: %s/%s, %s, ETA %s", p.Path, humanize.IBytes(safeUint64(p.BytesRead)), humanize.IBytes(safeUint64(p.TotalBytes)), rate, p.ETA.Round(time.Second)) } else { log.Infof("%s: %s/%s, %s", p.Path, humanize.IBytes(safeUint64(p.BytesRead)), humanize.IBytes(safeUint64(p.TotalBytes)), rate) } } } // manifestBaseURL returns the URL of the directory containing the // manifest, with a trailing slash. func manifestBaseURL(manifestURL string) (*url.URL, error) { baseURL, err := url.Parse(manifestURL) if err != nil { return nil, fmt.Errorf("fetch: invalid manifest URL: %w", err) } baseURL.Path = path.Dir(baseURL.Path) if !strings.HasSuffix(baseURL.Path, "/") { baseURL.Path += "/" } return baseURL, nil } // downloadManifestFiles downloads every file in the manifest, reporting // progress on the progress channel. func downloadManifestFiles( ctx context.Context, baseURL *url.URL, files []*mfer.MFFilePath, progress chan<- DownloadProgress, ) error { for _, f := range files { // Sanitize the path to prevent path traversal attacks localPath, err := sanitizePath(f.GetPath()) if err != nil { return fmt.Errorf("invalid path in manifest: %w", err) } fileURL := baseURL.String() + encodeFilePath(f.GetPath()) log.Infof("fetching %s", f.GetPath()) err = downloadFile(ctx, fileURL, localPath, f, progress) if err != nil { return fmt.Errorf("failed to download %s: %w", f.GetPath(), err) } } return nil } func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error { log.Debug("fetchManifestOperation()") if ctx.Args().Len() == 0 { return errURLRequired } inputURL := ctx.Args().Get(0) manifestURL, err := resolveManifestURL(inputURL) if err != nil { return fmt.Errorf("invalid URL: %w", err) } log.Infof("fetching manifest from %s", manifestURL) // Fetch manifest resp, err := httpGet(ctx.Context, manifestURL) if err != nil { return fmt.Errorf("failed to fetch manifest: %w", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return fmt.Errorf("failed to fetch manifest: %w %d", errHTTPStatus, resp.StatusCode) } // Parse manifest manifest, err := mfer.NewManifestFromReader(resp.Body) if err != nil { return fmt.Errorf("failed to parse manifest: %w", err) } files := manifest.Files() log.Infof("manifest contains %d files", len(files)) // Compute base URL (directory containing manifest) baseURL, err := manifestBaseURL(manifestURL) if err != nil { return err } // Calculate total bytes to download var totalBytes int64 for _, f := range files { totalBytes += f.GetSize() } // Create progress channel and start progress reporter goroutine progress := make(chan DownloadProgress, progressChanBuffer) done := make(chan struct{}) go reportDownloadProgress(progress, done) // Track download start time startTime := time.Now() // Download each file dlErr := downloadManifestFiles(ctx.Context, baseURL, files, progress) close(progress) <-done if dlErr != nil { return dlErr } // Print summary elapsed := time.Since(startTime) avgBytesPerSec := float64(totalBytes) / elapsed.Seconds() avgRate := formatBitrate(avgBytesPerSec * bitsPerByte) log.Infof("downloaded %d files (%s) in %.1fs (%s avg)", len(files), humanize.IBytes(safeUint64(totalBytes)), elapsed.Seconds(), avgRate) return nil } // encodeFilePath URL-encodes each segment of a file path while preserving slashes. func encodeFilePath(p string) string { segments := strings.Split(p, "/") for i, seg := range segments { segments[i] = url.PathEscape(seg) } return strings.Join(segments, "/") } // sanitizePath validates and sanitizes a file path from the manifest. // It prevents path traversal attacks and rejects unsafe paths. func sanitizePath(p string) (string, error) { // Reject empty paths if p == "" { return "", errEmptyPath } // Reject absolute paths if filepath.IsAbs(p) { return "", fmt.Errorf("%w: %s", errAbsolutePath, p) } // Clean the path to resolve . and .. cleaned := filepath.Clean(p) // Reject paths that escape the current directory if strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) || cleaned == ".." { return "", fmt.Errorf("%w: %s", errPathTraversal, p) } // Also check for absolute paths after cleaning (handles edge cases) if filepath.IsAbs(cleaned) { return "", fmt.Errorf("%w: %s", errAbsolutePath, p) } return cleaned, nil } // resolveManifestURL takes a URL and returns the manifest URL. // If the URL already ends with .mf, it's returned as-is. // Otherwise, index.mf is appended. func resolveManifestURL(inputURL string) (string, error) { parsed, err := url.Parse(inputURL) if err != nil { return "", fmt.Errorf("failed to parse URL: %w", err) } // Check if URL already ends with .mf if strings.HasSuffix(parsed.Path, ".mf") { return inputURL, nil } // Ensure path ends with / if !strings.HasSuffix(parsed.Path, "/") { parsed.Path += "/" } // Append index.mf parsed.Path += "index.mf" return parsed.String(), nil } // progressWriter wraps an io.Writer and reports progress to a channel. type progressWriter struct { w io.Writer path string total int64 written int64 startTime time.Time progress chan<- DownloadProgress } func (pw *progressWriter) Write(p []byte) (int, error) { n, err := pw.w.Write(p) pw.written += int64(n) if pw.progress != nil { var ( bytesPerSec float64 eta time.Duration ) elapsed := time.Since(pw.startTime) if elapsed > 0 && pw.written > 0 { bytesPerSec = float64(pw.written) / elapsed.Seconds() if bytesPerSec > 0 && pw.total > 0 { remainingBytes := pw.total - pw.written eta = time.Duration(float64(remainingBytes)/bytesPerSec) * time.Second } } sendProgress(pw.progress, DownloadProgress{ Path: pw.path, BytesRead: pw.written, TotalBytes: pw.total, BytesPerSec: bytesPerSec, ETA: eta, }) } return n, err } // formatBitrate formats a bits-per-second value with appropriate unit prefix. func formatBitrate(bps float64) string { switch { case bps >= bpsPerGbps: return fmt.Sprintf("%.1f Gbps", bps/bpsPerGbps) case bps >= bpsPerMbps: return fmt.Sprintf("%.1f Mbps", bps/bpsPerMbps) case bps >= bpsPerKbps: return fmt.Sprintf("%.1f Kbps", bps/bpsPerKbps) default: return fmt.Sprintf("%.0f bps", bps) } } // sendProgress sends a progress update without blocking. func sendProgress(ch chan<- DownloadProgress, p DownloadProgress) { select { case ch <- p: default: } } // tempPathFor computes the temporary download path for a local file. // For dotfiles, just append .tmp (they're already hidden); for regular // files, prefix with . and append .tmp. func tempPathFor(localPath string) string { dir := filepath.Dir(localPath) base := filepath.Base(localPath) var tmpName string if strings.HasPrefix(base, ".") { tmpName = base + ".tmp" } else { tmpName = "." + base + ".tmp" } if dir == "" || dir == "." { return tmpName } return filepath.Join(dir, tmpName) } // verifyDownloadedHash checks the computed sha256 digest against the // manifest entry's hashes; at least one must match. func verifyDownloadedHash(digest []byte, entry *mfer.MFFilePath) error { computed, err := multihash.Encode(digest, multihash.SHA2_256) if err != nil { return fmt.Errorf("failed to encode hash: %w", err) } for _, hash := range entry.GetHashes() { if bytes.Equal(computed, hash.GetMultiHash()) { return nil } } return errHashMismatch } // downloadFile downloads a URL to a local file path with hash verification. // It downloads to a temporary file, verifies the hash, then renames to the final path. // Progress is reported via the progress channel. func downloadFile( ctx context.Context, fileURL, localPath string, entry *mfer.MFFilePath, progress chan<- DownloadProgress, ) error { // Enforce the path invariant here rather than relying on the caller, // so every entry point to downloadFile gets the same treatment. localPath, err := sanitizePath(localPath) if err != nil { return fmt.Errorf("invalid path: %w", err) } // Create parent directories if needed dir := filepath.Dir(localPath) if dir != "" && dir != "." { err := os.MkdirAll(dir, dirPerms) if err != nil { return fmt.Errorf("failed to create directory %s: %w", dir, err) } } tmpPath := tempPathFor(localPath) // Fetch file resp, err := httpGet(ctx, fileURL) if err != nil { return fmt.Errorf("HTTP request failed: %w", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return fmt.Errorf("%w %d", errHTTPStatus, resp.StatusCode) } // Determine expected size expectedSize := entry.GetSize() totalBytes := resp.ContentLength if totalBytes < 0 { totalBytes = expectedSize } // Create temp file. // // G304: tmpPath is derived from localPath, which sanitizePath above // constrains lexically to a relative path that does not escape the // destination directory. That is a purely lexical guarantee: it does // not resolve symlinks, so a pre-existing symlink inside the // destination tree can still redirect this write outside of it // (tracked in issue #86). out, err := os.Create(tmpPath) //nolint:gosec // G304: see comment above if err != nil { return fmt.Errorf("failed to create temp file: %w", err) } // Set up hash computation h := sha256.New() // Create progress-reporting writer that also computes hash pw := &progressWriter{ w: io.MultiWriter(out, h), path: localPath, total: totalBytes, startTime: time.Now(), progress: progress, } // Copy content while hashing and reporting progress written, copyErr := io.Copy(pw, resp.Body) // Close file before checking errors (to flush writes) closeErr := out.Close() err = finishDownload( tmpPath, localPath, written, expectedSize, h.Sum(nil), entry, copyErr, closeErr) if err != nil { _ = os.Remove(tmpPath) return err } return nil } // finishDownload validates the copy result, verifies size and hash, and // moves the temp file into place. On error the caller removes tmpPath. func finishDownload( tmpPath, localPath string, written, expectedSize int64, digest []byte, entry *mfer.MFFilePath, copyErr, closeErr error, ) error { if copyErr != nil { return copyErr } if closeErr != nil { return closeErr } // Verify size if written != expectedSize { return fmt.Errorf("%w: expected %d bytes, got %d", errSizeMismatch, expectedSize, written) } // Verify hash against manifest (at least one must match) err := verifyDownloadedHash(digest, entry) if err != nil { return err } // Rename temp file to final path err = os.Rename(tmpPath, localPath) if err != nil { return fmt.Errorf("failed to rename temp file: %w", err) } return nil }