1.0 quality polish — code review, tests, bug fixes, documentation (#32)

Comprehensive quality pass targeting 1.0 release:

- Code review and refactoring
- Fix open bugs (#14, #16, #23)
- Expand test coverage
- Lint clean
- README update with build instructions (#9)
- Documentation improvements

Branched from `next` (active dev branch).

Reviewed-on: #32
Co-authored-by: clawbot <clawbot@noreply.example.org>
Co-committed-by: clawbot <clawbot@noreply.example.org>

internal/cli/check.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"encoding/hex"
 	"fmt"
+	"io"
 	"path/filepath"
 	"strings"
 	"time"
@@ -34,29 +35,32 @@ func findManifest(fs afero.Fs, dir string) (string, error) {
 func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
 	log.Debug("checkManifestOperation()")
 
-	var manifestPath string
-	var err error
+	manifestPath, err := mfa.resolveManifestArg(ctx)
+	if err != nil {
+		return fmt.Errorf("check: %w", err)
+	}
 
-	if ctx.Args().Len() > 0 {
-		arg := ctx.Args().Get(0)
-		// Check if arg is a directory or a file
-		info, statErr := mfa.Fs.Stat(arg)
-		if statErr == nil && info.IsDir() {
-			// It's a directory, look for manifest inside
-			manifestPath, err = findManifest(mfa.Fs, arg)
-			if err != nil {
-				return err
-			}
-		} else {
-			// Treat as a file path
-			manifestPath = arg
+	// URL manifests need to be downloaded to a temp file for the checker
+	if isHTTPURL(manifestPath) {
+		rc, fetchErr := mfa.openManifestReader(manifestPath)
+		if fetchErr != nil {
+			return fmt.Errorf("check: %w", fetchErr)
 		}
-	} else {
-		// No argument, look in current directory
-		manifestPath, err = findManifest(mfa.Fs, ".")
-		if err != nil {
-			return err
+		tmpFile, tmpErr := afero.TempFile(mfa.Fs, "", "mfer-manifest-*.mf")
+		if tmpErr != nil {
+			_ = rc.Close()
+			return fmt.Errorf("check: failed to create temp file: %w", tmpErr)
 		}
+		tmpPath := tmpFile.Name()
+		_, cpErr := io.Copy(tmpFile, rc)
+		_ = rc.Close()
+		_ = tmpFile.Close()
+		if cpErr != nil {
+			_ = mfa.Fs.Remove(tmpPath)
+			return fmt.Errorf("check: failed to download manifest: %w", cpErr)
+		}
+		defer func() { _ = mfa.Fs.Remove(tmpPath) }()
+		manifestPath = tmpPath
 	}
 
 	basePath := ctx.String("base")

internal/cli/export.go

@@ -0,0 +1,72 @@
+package cli
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/urfave/cli/v2"
+	"sneak.berlin/go/mfer/mfer"
+)
+
+// ExportEntry represents a single file entry in the exported JSON output.
+type ExportEntry struct {
+	Path   string   `json:"path"`
+	Size   int64    `json:"size"`
+	Hashes []string `json:"hashes"`
+	Mtime  *string  `json:"mtime,omitempty"`
+	Ctime  *string  `json:"ctime,omitempty"`
+}
+
+func (mfa *CLIApp) exportManifestOperation(ctx *cli.Context) error {
+	pathOrURL, err := mfa.resolveManifestArg(ctx)
+	if err != nil {
+		return fmt.Errorf("export: %w", err)
+	}
+
+	rc, err := mfa.openManifestReader(pathOrURL)
+	if err != nil {
+		return fmt.Errorf("export: %w", err)
+	}
+	defer func() { _ = rc.Close() }()
+
+	manifest, err := mfer.NewManifestFromReader(rc)
+	if err != nil {
+		return fmt.Errorf("export: failed to parse manifest: %w", err)
+	}
+
+	files := manifest.Files()
+	entries := make([]ExportEntry, 0, len(files))
+
+	for _, f := range files {
+		entry := ExportEntry{
+			Path:   f.Path,
+			Size:   f.Size,
+			Hashes: make([]string, 0, len(f.Hashes)),
+		}
+
+		for _, h := range f.Hashes {
+			entry.Hashes = append(entry.Hashes, hex.EncodeToString(h.MultiHash))
+		}
+
+		if f.Mtime != nil {
+			t := time.Unix(f.Mtime.Seconds, int64(f.Mtime.Nanos)).UTC().Format(time.RFC3339Nano)
+			entry.Mtime = &t
+		}
+		if f.Ctime != nil {
+			t := time.Unix(f.Ctime.Seconds, int64(f.Ctime.Nanos)).UTC().Format(time.RFC3339Nano)
+			entry.Ctime = &t
+		}
+
+		entries = append(entries, entry)
+	}
+
+	enc := json.NewEncoder(mfa.Stdout)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(entries); err != nil {
+		return fmt.Errorf("export: failed to encode JSON: %w", err)
+	}
+
+	return nil
+}

internal/cli/export_test.go

@@ -0,0 +1,137 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/mfer/mfer"
+)
+
+// buildTestManifest creates a manifest from in-memory files and returns its bytes.
+func buildTestManifest(t *testing.T, files map[string][]byte) []byte {
+	t.Helper()
+	sourceFs := afero.NewMemMapFs()
+	for path, content := range files {
+		require.NoError(t, sourceFs.MkdirAll("/", 0o755))
+		require.NoError(t, afero.WriteFile(sourceFs, "/"+path, content, 0o644))
+	}
+
+	opts := &mfer.ScannerOptions{Fs: sourceFs}
+	s := mfer.NewScannerWithOptions(opts)
+	require.NoError(t, s.EnumerateFS(sourceFs, "/", nil))
+
+	var buf bytes.Buffer
+	require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
+	return buf.Bytes()
+}
+
+func TestExportManifestOperation(t *testing.T) {
+	testFiles := map[string][]byte{
+		"hello.txt":    []byte("Hello, World!"),
+		"sub/file.txt": []byte("nested content"),
+	}
+	manifestData := buildTestManifest(t, testFiles)
+
+	// Write manifest to memfs
+	fs := afero.NewMemMapFs()
+	require.NoError(t, afero.WriteFile(fs, "/test.mf", manifestData, 0o644))
+
+	var stdout, stderr bytes.Buffer
+	exitCode := RunWithOptions(&RunOptions{
+		Appname: "mfer",
+		Args:    []string{"mfer", "export", "/test.mf"},
+		Stdin:   &bytes.Buffer{},
+		Stdout:  &stdout,
+		Stderr:  &stderr,
+		Fs:      fs,
+	})
+
+	require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
+
+	var entries []ExportEntry
+	require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries))
+	assert.Len(t, entries, 2)
+
+	// Verify entries have expected fields
+	pathSet := make(map[string]bool)
+	for _, e := range entries {
+		pathSet[e.Path] = true
+		assert.NotEmpty(t, e.Hashes, "entry %s should have hashes", e.Path)
+		assert.Greater(t, e.Size, int64(0), "entry %s should have positive size", e.Path)
+	}
+	assert.True(t, pathSet["hello.txt"])
+	assert.True(t, pathSet["sub/file.txt"])
+}
+
+func TestExportFromHTTPURL(t *testing.T) {
+	testFiles := map[string][]byte{
+		"a.txt": []byte("aaa"),
+	}
+	manifestData := buildTestManifest(t, testFiles)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/octet-stream")
+		_, _ = w.Write(manifestData)
+	}))
+	defer server.Close()
+
+	var stdout, stderr bytes.Buffer
+	exitCode := RunWithOptions(&RunOptions{
+		Appname: "mfer",
+		Args:    []string{"mfer", "export", server.URL + "/index.mf"},
+		Stdin:   &bytes.Buffer{},
+		Stdout:  &stdout,
+		Stderr:  &stderr,
+		Fs:      afero.NewMemMapFs(),
+	})
+
+	require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
+
+	var entries []ExportEntry
+	require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries))
+	assert.Len(t, entries, 1)
+	assert.Equal(t, "a.txt", entries[0].Path)
+}
+
+func TestListFromHTTPURL(t *testing.T) {
+	testFiles := map[string][]byte{
+		"one.txt": []byte("1"),
+		"two.txt": []byte("22"),
+	}
+	manifestData := buildTestManifest(t, testFiles)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(manifestData)
+	}))
+	defer server.Close()
+
+	var stdout, stderr bytes.Buffer
+	exitCode := RunWithOptions(&RunOptions{
+		Appname: "mfer",
+		Args:    []string{"mfer", "list", server.URL + "/index.mf"},
+		Stdin:   &bytes.Buffer{},
+		Stdout:  &stdout,
+		Stderr:  &stderr,
+		Fs:      afero.NewMemMapFs(),
+	})
+
+	require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
+	output := stdout.String()
+	assert.Contains(t, output, "one.txt")
+	assert.Contains(t, output, "two.txt")
+}
+
+func TestIsHTTPURL(t *testing.T) {
+	assert.True(t, isHTTPURL("http://example.com/manifest.mf"))
+	assert.True(t, isHTTPURL("https://example.com/manifest.mf"))
+	assert.False(t, isHTTPURL("/local/path.mf"))
+	assert.False(t, isHTTPURL("relative/path.mf"))
+	assert.False(t, isHTTPURL("ftp://example.com/file"))
+}

internal/cli/fetch.go

@@ -67,7 +67,7 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
 	// Compute base URL (directory containing manifest)
 	baseURL, err := url.Parse(manifestURL)
 	if err != nil {
-		return err
+		return fmt.Errorf("fetch: invalid manifest URL: %w", err)
 	}
 	baseURL.Path = path.Dir(baseURL.Path)
 	if !strings.HasSuffix(baseURL.Path, "/") {
@@ -267,7 +267,7 @@ func downloadFile(fileURL, localPath string, entry *mfer.MFFilePath, progress ch
 	dir := filepath.Dir(localPath)
 	if dir != "" && dir != "." {
 		if err := os.MkdirAll(dir, 0o755); err != nil {
-			return err
+			return fmt.Errorf("failed to create directory %s: %w", dir, err)
 		}
 	}
 
@@ -287,9 +287,9 @@ func downloadFile(fileURL, localPath string, entry *mfer.MFFilePath, progress ch
 	}
 
 	// Fetch file
-	resp, err := http.Get(fileURL)
+	resp, err := http.Get(fileURL) //nolint:gosec // URL constructed from manifest base
 	if err != nil {
-		return err
+		return fmt.Errorf("HTTP request failed: %w", err)
 	}
 	defer func() { _ = resp.Body.Close() }()
 
@@ -307,7 +307,7 @@ func downloadFile(fileURL, localPath string, entry *mfer.MFFilePath, progress ch
 	// Create temp file
 	out, err := os.Create(tmpPath)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to create temp file: %w", err)
 	}
 
 	// Set up hash computation

internal/cli/freshen.go

@@ -41,8 +41,8 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
 
 	basePath := ctx.String("base")
 	showProgress := ctx.Bool("progress")
-	includeDotfiles := ctx.Bool("IncludeDotfiles")
-	followSymlinks := ctx.Bool("FollowSymLinks")
+	includeDotfiles := ctx.Bool("include-dotfiles")
+	followSymlinks := ctx.Bool("follow-symlinks")
 
 	// Find manifest file
 	var manifestPath string
@@ -54,7 +54,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
 		if statErr == nil && info.IsDir() {
 			manifestPath, err = findManifest(mfa.Fs, arg)
 			if err != nil {
-				return err
+				return fmt.Errorf("freshen: %w", err)
 			}
 		} else {
 			manifestPath = arg
@@ -62,7 +62,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
 	} else {
 		manifestPath, err = findManifest(mfa.Fs, ".")
 		if err != nil {
-			return err
+			return fmt.Errorf("freshen: %w", err)
 		}
 	}
 
@@ -93,7 +93,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
 
 	absBase, err := filepath.Abs(basePath)
 	if err != nil {
-		return err
+		return fmt.Errorf("freshen: invalid base path: %w", err)
 	}
 
 	err = afero.Walk(mfa.Fs, absBase, func(path string, info fs.FileInfo, walkErr error) error {
@@ -104,7 +104,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
 		// Get relative path
 		relPath, err := filepath.Rel(absBase, path)
 		if err != nil {
-			return err
+			return fmt.Errorf("freshen: failed to compute relative path for %s: %w", path, err)
 		}
 
 		// Skip the manifest file itself
@@ -226,6 +226,9 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
 	var hashedBytes int64
 
 	builder := mfer.NewBuilder()
+	if ctx.Bool("include-timestamps") {
+		builder.SetIncludeTimestamps(true)
+	}
 
 	// Set up signing options if sign-key is provided
 	if signKey := ctx.String("sign-key"); signKey != "" {

internal/cli/gen.go

@@ -20,9 +20,10 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
 	log.Debug("generateManifestOperation()")
 
 	opts := &mfer.ScannerOptions{
-		IncludeDotfiles: ctx.Bool("IncludeDotfiles"),
-		FollowSymLinks:  ctx.Bool("FollowSymLinks"),
-		Fs:              mfa.Fs,
+		IncludeDotfiles:   ctx.Bool("include-dotfiles"),
+		FollowSymLinks:    ctx.Bool("follow-symlinks"),
+		IncludeTimestamps: ctx.Bool("include-timestamps"),
+		Fs:                mfa.Fs,
 	}
 
 	// Set seed for deterministic UUID if provided
@@ -65,7 +66,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
 	if args.Len() == 0 {
 		// Default to current directory
 		if err := s.EnumeratePath(".", enumProgress); err != nil {
-			return err
+			return fmt.Errorf("generate: failed to enumerate current directory: %w", err)
 		}
 	} else {
 		// Collect and validate all paths first
@@ -74,7 +75,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
 			inputPath := args.Get(i)
 			ap, err := filepath.Abs(inputPath)
 			if err != nil {
-				return err
+				return fmt.Errorf("generate: invalid path %q: %w", inputPath, err)
 			}
 			// Validate path exists before adding to list
 			if exists, _ := afero.Exists(mfa.Fs, ap); !exists {
@@ -84,7 +85,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
 			paths = append(paths, ap)
 		}
 		if err := s.EnumeratePaths(enumProgress, paths...); err != nil {
-			return err
+			return fmt.Errorf("generate: failed to enumerate paths: %w", err)
 		}
 	}
 	enumWg.Wait()

internal/cli/list.go

@@ -16,32 +16,20 @@ func (mfa *CLIApp) listManifestOperation(ctx *cli.Context) error {
 	longFormat := ctx.Bool("long")
 	print0 := ctx.Bool("print0")
 
-	// Find manifest file
-	var manifestPath string
-	var err error
-
-	if ctx.Args().Len() > 0 {
-		arg := ctx.Args().Get(0)
-		info, statErr := mfa.Fs.Stat(arg)
-		if statErr == nil && info.IsDir() {
-			manifestPath, err = findManifest(mfa.Fs, arg)
-			if err != nil {
-				return err
-			}
-		} else {
-			manifestPath = arg
-		}
-	} else {
-		manifestPath, err = findManifest(mfa.Fs, ".")
-		if err != nil {
-			return err
-		}
+	pathOrURL, err := mfa.resolveManifestArg(ctx)
+	if err != nil {
+		return fmt.Errorf("list: %w", err)
 	}
 
-	// Load manifest
-	manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath)
+	rc, err := mfa.openManifestReader(pathOrURL)
 	if err != nil {
-		return fmt.Errorf("failed to load manifest: %w", err)
+		return fmt.Errorf("list: %w", err)
+	}
+	defer func() { _ = rc.Close() }()
+
+	manifest, err := mfer.NewManifestFromReader(rc)
+	if err != nil {
+		return fmt.Errorf("list: failed to parse manifest: %w", err)
 	}
 
 	files := manifest.Files()

internal/cli/manifest_loader.go

@@ -0,0 +1,56 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/urfave/cli/v2"
+)
+
+// isHTTPURL returns true if the string starts with http:// or https://.
+func isHTTPURL(s string) bool {
+	return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
+}
+
+// openManifestReader opens a manifest from a path or URL and returns a ReadCloser.
+// The caller must close the returned reader.
+func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
+	if isHTTPURL(pathOrURL) {
+		client := &http.Client{Timeout: 30 * time.Second}
+		resp, err := client.Get(pathOrURL) //nolint:gosec // user-provided URL is intentional
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch %s: %w", pathOrURL, err)
+		}
+		if resp.StatusCode != http.StatusOK {
+			_ = resp.Body.Close()
+			return nil, fmt.Errorf("failed to fetch %s: HTTP %d", pathOrURL, resp.StatusCode)
+		}
+		return resp.Body, nil
+	}
+	f, err := mfa.Fs.Open(pathOrURL)
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+// resolveManifestArg resolves the manifest path from CLI arguments.
+// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf.
+// If no argument is given, the current directory is searched.
+func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
+	if ctx.Args().Len() > 0 {
+		arg := ctx.Args().Get(0)
+		if isHTTPURL(arg) {
+			return arg, nil
+		}
+		info, statErr := mfa.Fs.Stat(arg)
+		if statErr == nil && info.IsDir() {
+			return findManifest(mfa.Fs, arg)
+		}
+		return arg, nil
+	}
+	return findManifest(mfa.Fs, ".")
+}

internal/cli/mfer.go

@@ -123,14 +123,15 @@ func (mfa *CLIApp) run(args []string) {
 				},
 				Flags: append(commonFlags(),
 					&cli.BoolFlag{
-						Name:    "FollowSymLinks",
-						Aliases: []string{"follow-symlinks"},
+						Name:    "follow-symlinks",
+						Aliases: []string{"L"},
 						Usage:   "Resolve encountered symlinks",
 					},
 					&cli.BoolFlag{
-						Name:    "IncludeDotfiles",
-						Aliases: []string{"include-dotfiles"},
-						Usage:   "Include dot (hidden) files (excluded by default)",
+						Name:    "include-dotfiles",
+						Aliases: []string{"IncludeDotfiles"},
+
+						Usage: "Include dot (hidden) files (excluded by default)",
 					},
 					&cli.StringFlag{
 						Name:    "output",
@@ -159,6 +160,10 @@ func (mfa *CLIApp) run(args []string) {
 						Usage:   "Seed value for deterministic manifest UUID",
 						EnvVars: []string{"MFER_SEED"},
 					},
+					&cli.BoolFlag{
+						Name:  "include-timestamps",
+						Usage: "Include createdAt timestamp in manifest (omitted by default for determinism)",
+					},
 				),
 			},
 			{
@@ -211,14 +216,15 @@ func (mfa *CLIApp) run(args []string) {
 						Usage:   "Base directory for resolving relative paths",
 					},
 					&cli.BoolFlag{
-						Name:    "FollowSymLinks",
-						Aliases: []string{"follow-symlinks"},
+						Name:    "follow-symlinks",
+						Aliases: []string{"L"},
 						Usage:   "Resolve encountered symlinks",
 					},
 					&cli.BoolFlag{
-						Name:    "IncludeDotfiles",
-						Aliases: []string{"include-dotfiles"},
-						Usage:   "Include dot (hidden) files (excluded by default)",
+						Name:    "include-dotfiles",
+						Aliases: []string{"IncludeDotfiles"},
+
+						Usage: "Include dot (hidden) files (excluded by default)",
 					},
 					&cli.BoolFlag{
 						Name:    "progress",
@@ -231,8 +237,20 @@ func (mfa *CLIApp) run(args []string) {
 						Usage:   "GPG key ID to sign the manifest with",
 						EnvVars: []string{"MFER_SIGN_KEY"},
 					},
+					&cli.BoolFlag{
+						Name:  "include-timestamps",
+						Usage: "Include createdAt timestamp in manifest (omitted by default for determinism)",
+					},
 				),
 			},
+			{
+				Name:      "export",
+				Usage:     "Export manifest contents as JSON",
+				ArgsUsage: "[manifest file or URL]",
+				Action: func(c *cli.Context) error {
+					return mfa.exportManifestOperation(c)
+				},
+			},
 			{
 				Name:  "version",
 				Usage: "Show version",
@@ -274,7 +292,7 @@ func (mfa *CLIApp) run(args []string) {
 		},
 	}
 
-	mfa.app.HideVersion = true
+	mfa.app.HideVersion = false
 	err := mfa.app.Run(args)
 	if err != nil {
 		mfa.exitCode = 1