diff --git a/internal/cli/check.go b/internal/cli/check.go index a2d0cdf..7327474 100644 --- a/internal/cli/check.go +++ b/internal/cli/check.go @@ -3,6 +3,7 @@ package cli import ( "encoding/hex" "fmt" + "io" "path/filepath" "strings" "time" @@ -34,29 +35,32 @@ func findManifest(fs afero.Fs, dir string) (string, error) { func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error { log.Debug("checkManifestOperation()") - var manifestPath string - var err error + manifestPath, err := mfa.resolveManifestArg(ctx) + if err != nil { + return fmt.Errorf("check: %w", err) + } - if ctx.Args().Len() > 0 { - arg := ctx.Args().Get(0) - // Check if arg is a directory or a file - info, statErr := mfa.Fs.Stat(arg) - if statErr == nil && info.IsDir() { - // It's a directory, look for manifest inside - manifestPath, err = findManifest(mfa.Fs, arg) - if err != nil { - return err - } - } else { - // Treat as a file path - manifestPath = arg + // URL manifests need to be downloaded to a temp file for the checker + if isHTTPURL(manifestPath) { + rc, fetchErr := mfa.openManifestReader(manifestPath) + if fetchErr != nil { + return fmt.Errorf("check: %w", fetchErr) } - } else { - // No argument, look in current directory - manifestPath, err = findManifest(mfa.Fs, ".") - if err != nil { - return err + tmpFile, tmpErr := afero.TempFile(mfa.Fs, "", "mfer-manifest-*.mf") + if tmpErr != nil { + _ = rc.Close() + return fmt.Errorf("check: failed to create temp file: %w", tmpErr) } + tmpPath := tmpFile.Name() + _, cpErr := io.Copy(tmpFile, rc) + _ = rc.Close() + _ = tmpFile.Close() + if cpErr != nil { + _ = mfa.Fs.Remove(tmpPath) + return fmt.Errorf("check: failed to download manifest: %w", cpErr) + } + defer func() { _ = mfa.Fs.Remove(tmpPath) }() + manifestPath = tmpPath } basePath := ctx.String("base") diff --git a/internal/cli/export.go b/internal/cli/export.go new file mode 100644 index 0000000..aca8ebf --- /dev/null +++ b/internal/cli/export.go @@ -0,0 +1,72 @@ +package cli + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "time" + + "github.com/urfave/cli/v2" + "sneak.berlin/go/mfer/mfer" +) + +// ExportEntry represents a single file entry in the exported JSON output. +type ExportEntry struct { + Path string `json:"path"` + Size int64 `json:"size"` + Hashes []string `json:"hashes"` + Mtime *string `json:"mtime,omitempty"` + Ctime *string `json:"ctime,omitempty"` +} + +func (mfa *CLIApp) exportManifestOperation(ctx *cli.Context) error { + pathOrURL, err := mfa.resolveManifestArg(ctx) + if err != nil { + return fmt.Errorf("export: %w", err) + } + + rc, err := mfa.openManifestReader(pathOrURL) + if err != nil { + return fmt.Errorf("export: %w", err) + } + defer func() { _ = rc.Close() }() + + manifest, err := mfer.NewManifestFromReader(rc) + if err != nil { + return fmt.Errorf("export: failed to parse manifest: %w", err) + } + + files := manifest.Files() + entries := make([]ExportEntry, 0, len(files)) + + for _, f := range files { + entry := ExportEntry{ + Path: f.Path, + Size: f.Size, + Hashes: make([]string, 0, len(f.Hashes)), + } + + for _, h := range f.Hashes { + entry.Hashes = append(entry.Hashes, hex.EncodeToString(h.MultiHash)) + } + + if f.Mtime != nil { + t := time.Unix(f.Mtime.Seconds, int64(f.Mtime.Nanos)).UTC().Format(time.RFC3339Nano) + entry.Mtime = &t + } + if f.Ctime != nil { + t := time.Unix(f.Ctime.Seconds, int64(f.Ctime.Nanos)).UTC().Format(time.RFC3339Nano) + entry.Ctime = &t + } + + entries = append(entries, entry) + } + + enc := json.NewEncoder(mfa.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(entries); err != nil { + return fmt.Errorf("export: failed to encode JSON: %w", err) + } + + return nil +} diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go new file mode 100644 index 0000000..6d1f87d --- /dev/null +++ b/internal/cli/export_test.go @@ -0,0 +1,137 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/mfer/mfer" +) + +// buildTestManifest creates a manifest from in-memory files and returns its bytes. +func buildTestManifest(t *testing.T, files map[string][]byte) []byte { + t.Helper() + sourceFs := afero.NewMemMapFs() + for path, content := range files { + require.NoError(t, sourceFs.MkdirAll("/", 0o755)) + require.NoError(t, afero.WriteFile(sourceFs, "/"+path, content, 0o644)) + } + + opts := &mfer.ScannerOptions{Fs: sourceFs} + s := mfer.NewScannerWithOptions(opts) + require.NoError(t, s.EnumerateFS(sourceFs, "/", nil)) + + var buf bytes.Buffer + require.NoError(t, s.ToManifest(context.Background(), &buf, nil)) + return buf.Bytes() +} + +func TestExportManifestOperation(t *testing.T) { + testFiles := map[string][]byte{ + "hello.txt": []byte("Hello, World!"), + "sub/file.txt": []byte("nested content"), + } + manifestData := buildTestManifest(t, testFiles) + + // Write manifest to memfs + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/test.mf", manifestData, 0o644)) + + var stdout, stderr bytes.Buffer + exitCode := RunWithOptions(&RunOptions{ + Appname: "mfer", + Args: []string{"mfer", "export", "/test.mf"}, + Stdin: &bytes.Buffer{}, + Stdout: &stdout, + Stderr: &stderr, + Fs: fs, + }) + + require.Equal(t, 0, exitCode, "stderr: %s", stderr.String()) + + var entries []ExportEntry + require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries)) + assert.Len(t, entries, 2) + + // Verify entries have expected fields + pathSet := make(map[string]bool) + for _, e := range entries { + pathSet[e.Path] = true + assert.NotEmpty(t, e.Hashes, "entry %s should have hashes", e.Path) + assert.Greater(t, e.Size, int64(0), "entry %s should have positive size", e.Path) + } + assert.True(t, pathSet["hello.txt"]) + assert.True(t, pathSet["sub/file.txt"]) +} + +func TestExportFromHTTPURL(t *testing.T) { + testFiles := map[string][]byte{ + "a.txt": []byte("aaa"), + } + manifestData := buildTestManifest(t, testFiles) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(manifestData) + })) + defer server.Close() + + var stdout, stderr bytes.Buffer + exitCode := RunWithOptions(&RunOptions{ + Appname: "mfer", + Args: []string{"mfer", "export", server.URL + "/index.mf"}, + Stdin: &bytes.Buffer{}, + Stdout: &stdout, + Stderr: &stderr, + Fs: afero.NewMemMapFs(), + }) + + require.Equal(t, 0, exitCode, "stderr: %s", stderr.String()) + + var entries []ExportEntry + require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries)) + assert.Len(t, entries, 1) + assert.Equal(t, "a.txt", entries[0].Path) +} + +func TestListFromHTTPURL(t *testing.T) { + testFiles := map[string][]byte{ + "one.txt": []byte("1"), + "two.txt": []byte("22"), + } + manifestData := buildTestManifest(t, testFiles) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(manifestData) + })) + defer server.Close() + + var stdout, stderr bytes.Buffer + exitCode := RunWithOptions(&RunOptions{ + Appname: "mfer", + Args: []string{"mfer", "list", server.URL + "/index.mf"}, + Stdin: &bytes.Buffer{}, + Stdout: &stdout, + Stderr: &stderr, + Fs: afero.NewMemMapFs(), + }) + + require.Equal(t, 0, exitCode, "stderr: %s", stderr.String()) + output := stdout.String() + assert.Contains(t, output, "one.txt") + assert.Contains(t, output, "two.txt") +} + +func TestIsHTTPURL(t *testing.T) { + assert.True(t, isHTTPURL("http://example.com/manifest.mf")) + assert.True(t, isHTTPURL("https://example.com/manifest.mf")) + assert.False(t, isHTTPURL("/local/path.mf")) + assert.False(t, isHTTPURL("relative/path.mf")) + assert.False(t, isHTTPURL("ftp://example.com/file")) +} diff --git a/internal/cli/fetch.go b/internal/cli/fetch.go index ade2080..677b65a 100644 --- a/internal/cli/fetch.go +++ b/internal/cli/fetch.go @@ -67,7 +67,7 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error { // Compute base URL (directory containing manifest) baseURL, err := url.Parse(manifestURL) if err != nil { - return err + return fmt.Errorf("fetch: invalid manifest URL: %w", err) } baseURL.Path = path.Dir(baseURL.Path) if !strings.HasSuffix(baseURL.Path, "/") { @@ -267,7 +267,7 @@ func downloadFile(fileURL, localPath string, entry *mfer.MFFilePath, progress ch dir := filepath.Dir(localPath) if dir != "" && dir != "." { if err := os.MkdirAll(dir, 0o755); err != nil { - return err + return fmt.Errorf("failed to create directory %s: %w", dir, err) } } @@ -287,9 +287,9 @@ func downloadFile(fileURL, localPath string, entry *mfer.MFFilePath, progress ch } // Fetch file - resp, err := http.Get(fileURL) + resp, err := http.Get(fileURL) //nolint:gosec // URL constructed from manifest base if err != nil { - return err + return fmt.Errorf("HTTP request failed: %w", err) } defer func() { _ = resp.Body.Close() }() @@ -307,7 +307,7 @@ func downloadFile(fileURL, localPath string, entry *mfer.MFFilePath, progress ch // Create temp file out, err := os.Create(tmpPath) if err != nil { - return err + return fmt.Errorf("failed to create temp file: %w", err) } // Set up hash computation diff --git a/internal/cli/freshen.go b/internal/cli/freshen.go index 095c777..a078ee5 100644 --- a/internal/cli/freshen.go +++ b/internal/cli/freshen.go @@ -54,7 +54,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error { if statErr == nil && info.IsDir() { manifestPath, err = findManifest(mfa.Fs, arg) if err != nil { - return err + return fmt.Errorf("freshen: %w", err) } } else { manifestPath = arg @@ -62,7 +62,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error { } else { manifestPath, err = findManifest(mfa.Fs, ".") if err != nil { - return err + return fmt.Errorf("freshen: %w", err) } } @@ -93,7 +93,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error { absBase, err := filepath.Abs(basePath) if err != nil { - return err + return fmt.Errorf("freshen: invalid base path: %w", err) } err = afero.Walk(mfa.Fs, absBase, func(path string, info fs.FileInfo, walkErr error) error { @@ -104,7 +104,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error { // Get relative path relPath, err := filepath.Rel(absBase, path) if err != nil { - return err + return fmt.Errorf("freshen: failed to compute relative path for %s: %w", path, err) } // Skip the manifest file itself diff --git a/internal/cli/gen.go b/internal/cli/gen.go index 8d19c4c..f424a6f 100644 --- a/internal/cli/gen.go +++ b/internal/cli/gen.go @@ -66,7 +66,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error { if args.Len() == 0 { // Default to current directory if err := s.EnumeratePath(".", enumProgress); err != nil { - return err + return fmt.Errorf("generate: failed to enumerate current directory: %w", err) } } else { // Collect and validate all paths first @@ -75,7 +75,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error { inputPath := args.Get(i) ap, err := filepath.Abs(inputPath) if err != nil { - return err + return fmt.Errorf("generate: invalid path %q: %w", inputPath, err) } // Validate path exists before adding to list if exists, _ := afero.Exists(mfa.Fs, ap); !exists { @@ -85,7 +85,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error { paths = append(paths, ap) } if err := s.EnumeratePaths(enumProgress, paths...); err != nil { - return err + return fmt.Errorf("generate: failed to enumerate paths: %w", err) } } enumWg.Wait() diff --git a/internal/cli/list.go b/internal/cli/list.go index b89c236..66031d7 100644 --- a/internal/cli/list.go +++ b/internal/cli/list.go @@ -16,32 +16,20 @@ func (mfa *CLIApp) listManifestOperation(ctx *cli.Context) error { longFormat := ctx.Bool("long") print0 := ctx.Bool("print0") - // Find manifest file - var manifestPath string - var err error - - if ctx.Args().Len() > 0 { - arg := ctx.Args().Get(0) - info, statErr := mfa.Fs.Stat(arg) - if statErr == nil && info.IsDir() { - manifestPath, err = findManifest(mfa.Fs, arg) - if err != nil { - return err - } - } else { - manifestPath = arg - } - } else { - manifestPath, err = findManifest(mfa.Fs, ".") - if err != nil { - return err - } + pathOrURL, err := mfa.resolveManifestArg(ctx) + if err != nil { + return fmt.Errorf("list: %w", err) } - // Load manifest - manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath) + rc, err := mfa.openManifestReader(pathOrURL) if err != nil { - return fmt.Errorf("failed to load manifest: %w", err) + return fmt.Errorf("list: %w", err) + } + defer func() { _ = rc.Close() }() + + manifest, err := mfer.NewManifestFromReader(rc) + if err != nil { + return fmt.Errorf("list: failed to parse manifest: %w", err) } files := manifest.Files() diff --git a/internal/cli/manifest_loader.go b/internal/cli/manifest_loader.go new file mode 100644 index 0000000..5d575d8 --- /dev/null +++ b/internal/cli/manifest_loader.go @@ -0,0 +1,54 @@ +package cli + +import ( + "fmt" + "io" + "net/http" + "strings" + + "github.com/urfave/cli/v2" +) + +// isHTTPURL returns true if the string starts with http:// or https://. +func isHTTPURL(s string) bool { + return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") +} + +// openManifestReader opens a manifest from a path or URL and returns a ReadCloser. +// The caller must close the returned reader. +func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) { + if isHTTPURL(pathOrURL) { + resp, err := http.Get(pathOrURL) //nolint:gosec // user-provided URL is intentional + if err != nil { + return nil, fmt.Errorf("failed to fetch %s: %w", pathOrURL, err) + } + if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() + return nil, fmt.Errorf("failed to fetch %s: HTTP %d", pathOrURL, resp.StatusCode) + } + return resp.Body, nil + } + f, err := mfa.Fs.Open(pathOrURL) + if err != nil { + return nil, err + } + return f, nil +} + +// resolveManifestArg resolves the manifest path from CLI arguments. +// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf. +// If no argument is given, the current directory is searched. +func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) { + if ctx.Args().Len() > 0 { + arg := ctx.Args().Get(0) + if isHTTPURL(arg) { + return arg, nil + } + info, statErr := mfa.Fs.Stat(arg) + if statErr == nil && info.IsDir() { + return findManifest(mfa.Fs, arg) + } + return arg, nil + } + return findManifest(mfa.Fs, ".") +} diff --git a/internal/cli/mfer.go b/internal/cli/mfer.go index 8c482d7..ed399f2 100644 --- a/internal/cli/mfer.go +++ b/internal/cli/mfer.go @@ -241,6 +241,14 @@ func (mfa *CLIApp) run(args []string) { }, ), }, + { + Name: "export", + Usage: "Export manifest contents as JSON", + ArgsUsage: "[manifest file or URL]", + Action: func(c *cli.Context) error { + return mfa.exportManifestOperation(c) + }, + }, { Name: "version", Usage: "Show version", @@ -282,7 +290,7 @@ func (mfa *CLIApp) run(args []string) { }, } - mfa.app.HideVersion = true + mfa.app.HideVersion = false err := mfa.app.Run(args) if err != nil { mfa.exitCode = 1 diff --git a/mfer/builder.go b/mfer/builder.go index facf173..0b2262f 100644 --- a/mfer/builder.go +++ b/mfer/builder.go @@ -196,7 +196,7 @@ func (b *Builder) FileCount() int { // Returns an error if path is empty, size is negative, or hash is nil/empty. func (b *Builder) AddFileWithHash(path RelFilePath, size FileSize, mtime ModTime, hash Multihash) error { if err := ValidatePath(string(path)); err != nil { - return err + return fmt.Errorf("add file: %w", err) } if size < 0 { return errors.New("size cannot be negative") @@ -264,15 +264,18 @@ func (b *Builder) Build(w io.Writer) error { // Generate outer wrapper if err := m.generateOuter(); err != nil { - return err + return fmt.Errorf("build: generate outer: %w", err) } // Generate final output if err := m.generate(); err != nil { - return err + return fmt.Errorf("build: generate: %w", err) } // Write to output _, err := w.Write(m.output.Bytes()) - return err + if err != nil { + return fmt.Errorf("build: write output: %w", err) + } + return nil } diff --git a/mfer/deserialize.go b/mfer/deserialize.go index 878bf8f..f8de802 100644 --- a/mfer/deserialize.go +++ b/mfer/deserialize.go @@ -44,7 +44,7 @@ func (m *manifest) deserializeInner() error { // Verify hash of compressed data before decompression h := sha256.New() if _, err := h.Write(m.pbOuter.InnerMessage); err != nil { - return err + return fmt.Errorf("deserialize: hash write: %w", err) } sha256Hash := h.Sum(nil) if !bytes.Equal(sha256Hash, m.pbOuter.Sha256) { @@ -72,7 +72,7 @@ func (m *manifest) deserializeInner() error { zr, err := zstd.NewReader(bb) if err != nil { - return err + return fmt.Errorf("deserialize: zstd reader: %w", err) } defer zr.Close() @@ -85,7 +85,7 @@ func (m *manifest) deserializeInner() error { limitedReader := io.LimitReader(zr, maxSize) dat, err := io.ReadAll(limitedReader) if err != nil { - return err + return fmt.Errorf("deserialize: decompress: %w", err) } if int64(len(dat)) >= MaxDecompressedSize { return fmt.Errorf("decompressed data exceeds maximum allowed size of %d bytes", MaxDecompressedSize) @@ -100,7 +100,7 @@ func (m *manifest) deserializeInner() error { // Deserialize inner message m.pbInner = new(MFFile) if err := proto.Unmarshal(dat, m.pbInner); err != nil { - return err + return fmt.Errorf("deserialize: unmarshal inner: %w", err) } // Validate inner UUID diff --git a/mfer/serialize.go b/mfer/serialize.go index 3d712a6..b60c1c0 100644 --- a/mfer/serialize.go +++ b/mfer/serialize.go @@ -34,12 +34,12 @@ func (m *manifest) generate() error { } dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter) if err != nil { - return err + return fmt.Errorf("serialize: marshal outer: %w", err) } m.output = bytes.NewBuffer([]byte(MAGIC)) _, err = m.output.Write(dat) if err != nil { - return err + return fmt.Errorf("serialize: write output: %w", err) } return nil } @@ -60,18 +60,18 @@ func (m *manifest) generateOuter() error { innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner) if err != nil { - return err + return fmt.Errorf("serialize: marshal inner: %w", err) } // Compress the inner data idc := new(bytes.Buffer) zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) if err != nil { - return err + return fmt.Errorf("serialize: create compressor: %w", err) } _, err = zw.Write(innerData) if err != nil { - return err + return fmt.Errorf("serialize: compress: %w", err) } _ = zw.Close() @@ -80,7 +80,7 @@ func (m *manifest) generateOuter() error { // Hash the compressed data for integrity verification before decompression h := sha256.New() if _, err := h.Write(compressedData); err != nil { - return err + return fmt.Errorf("serialize: hash write: %w", err) } sha256Hash := h.Sum(nil)