Sign and verify manifests in Go with OpenPGP instead of running gpg (closes #181)

mfer ran the gpg binary to sign, export keys and verify, so signing and
loading signed manifests failed wherever gpg is missing. It now uses
github.com/ProtonMail/go-crypto/openpgp. --sign-key and MFER_SIGN_KEY
name a file holding one OpenPGP secret key; a protected key's passphrase
comes from MFER_SIGN_KEY_PASSPHRASE or a prompt on the terminal.
Verification keeps the rules of the --require-signature fix: one primary
key in the embedded block, counted from its packets so that keys the
library skips count too, exactly one signature, made by that key or one
of its subkeys, and signer equal to its fingerprint. Tests make their
keys in process.

Model: opus-5-5

internal/cli/check.go

@@ -21,7 +21,7 @@ import (
 	"sneak.berlin/go/mfer/mfer"
 )
 
-// fingerprintHexLen is the length of a full GPG key fingerprint in hex
+// fingerprintHexLen is the length of a full OpenPGP key fingerprint in hex
 // characters.
 const fingerprintHexLen = 40
 
@@ -320,7 +320,6 @@ func (mfa *CLIApp) checkManifestOperation(
 	log.Infof("checking manifest %s with base %s", manifestPath, basePath)
 
 	// Create checker
-	//nolint:contextcheck // mfer loads a manifest without a context
 	chk, err := mfer.NewChecker(&mfer.CheckerOptions{
 		ManifestPath: manifestPath,
 		BasePath:     basePath,

internal/cli/entry_test.go

@@ -654,11 +654,10 @@ func runCheckAfterRewrite(t *testing.T, rewritten, msg string) {
 
 // TestCheckRequireSignatureRefusesOtherSigningKey runs check
 // --require-signature on a manifest signed by another key whose embedded
-// public key block also holds the required key. check must refuse it. It
-// needs gpg and is skipped without it, as the other signing tests are.
-//
-//nolint:paralleltest // signedManifest calls t.Setenv, which bars t.Parallel
+// public key block also holds the required key. check must refuse it.
 func TestCheckRequireSignatureRefusesOtherSigningKey(t *testing.T) {
+	t.Parallel()
+
 	content := []byte("signed file")
 	manifest, required := manifestSignedByAnotherKey(t,
 		map[string][]byte{testFileTxt: content})

internal/cli/errmsg_test.go

@@ -4,14 +4,18 @@ package cli
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"net/http"
 	"net/http/httptest"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"slices"
+	"strings"
 	"testing"
 
+	"github.com/ProtonMail/go-crypto/openpgp"
+	"github.com/ProtonMail/go-crypto/openpgp/armor"
+	"github.com/ProtonMail/go-crypto/openpgp/packet"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -108,11 +112,10 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
 // manifest. The signing key's fingerprint is whatever the generated key
 // produced, so it is read back from the checker and substituted into the
 // expected string; the required signer is a fixed value that cannot match
-// it. Requires gpg and is skipped where it is absent, as the other signing
-// tests are.
-//
-//nolint:paralleltest // signedManifest calls t.Setenv, which bars t.Parallel
+// it.
 func TestSignerMismatchMessage(t *testing.T) {
+	t.Parallel()
+
 	chk := signedChecker(t,
 		signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")}))
 
@@ -123,43 +126,42 @@ func TestSignerMismatchMessage(t *testing.T) {
 			" does not match required "+msgFpB)
 }
 
-// signedManifest returns a manifest of files signed by a throwaway GPG key
-// generated in a temporary GNUPGHOME, which it leaves set for the rest of
-// the test.
+// testSecretKey returns a new OpenPGP key with its secret key, armored, as
+// gpg --export-secret-keys --armor writes it, and the key's fingerprint.
+// The key is protected by passphrase unless that is nil. It is an Ed25519
+// key, which is quick to make.
+func testSecretKey(t *testing.T, passphrase []byte) ([]byte, string) {
+	t.Helper()
+
+	key, err := openpgp.NewEntity("MFER Test Key", "", "test@mfer.test",
+		&packet.Config{Algorithm: packet.PubKeyAlgoEdDSA})
+	require.NoError(t, err)
+
+	if passphrase != nil {
+		require.NoError(t, key.EncryptPrivateKeys(passphrase, nil))
+	}
+
+	var buf bytes.Buffer
+
+	w, err := armor.Encode(&buf, openpgp.PrivateKeyType, nil)
+	require.NoError(t, err)
+	require.NoError(t, key.SerializePrivateWithoutSigning(w, nil))
+	require.NoError(t, w.Close())
+
+	return buf.Bytes(), strings.ToUpper(hex.EncodeToString(key.PrimaryKey.Fingerprint))
+}
+
+// signedManifest returns a manifest of files signed by a new OpenPGP key.
 func signedManifest(t *testing.T, files map[string][]byte) []byte {
 	t.Helper()
 
-	_, err := exec.LookPath("gpg")
-	if err != nil {
-		t.Skip("gpg not installed, skipping signing test")
-	}
-
-	gpgHome := t.TempDir()
-	params := "%no-protection\n" +
-		"Key-Type: RSA\nKey-Length: 2048\n" +
-		"Name-Real: MFER Test Key\nName-Email: test@mfer.test\n" +
-		"Expire-Date: 0\n%commit\n"
-	paramsFile := filepath.Join(gpgHome, "key-params")
-	require.NoError(t, os.WriteFile(paramsFile, []byte(params), 0o600))
-
-	//nolint:gosec // paramsFile is a test-controlled path inside t.TempDir()
-	cmd := exec.CommandContext(context.Background(), "gpg",
-		"--batch", "--gen-key", paramsFile)
-
-	cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
-
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Skipf("failed to generate test GPG key: %v: %s", err, out)
-	}
-
-	t.Setenv("GNUPGHOME", gpgHome)
+	secretKey, _ := testSecretKey(t, nil)
 
 	b := mfer.NewBuilder()
-	b.SetSigningOptions(&mfer.SigningOptions{KeyID: mfer.GPGKeyID("test@mfer.test")})
+	b.SetSigningOptions(&mfer.SigningOptions{SecretKey: secretKey})
 
 	for path, content := range files {
-		_, err = b.AddFile(mfer.RelFilePath(path), mfer.FileSize(len(content)),
+		_, err := b.AddFile(mfer.RelFilePath(path), mfer.FileSize(len(content)),
 			mfer.ModTime{}, 0, bytes.NewReader(content), nil)
 		require.NoError(t, err)
 	}
@@ -190,8 +192,8 @@ func signedChecker(t *testing.T, manifest []byte) *mfer.Checker {
 }
 
 // manifestSignedByAnotherKey returns a manifest of files and the
-// fingerprint of a throwaway key, the required key, that did not sign it.
-// The manifest is signed by a second throwaway key; its embedded public key
+// fingerprint of a new key, the required key, that did not sign it.
+// The manifest is signed by a second new key; its embedded public key
 // block holds the required key followed by the second key, and its signer
 // field names the required key.
 func manifestSignedByAnotherKey(
@@ -207,8 +209,10 @@ func manifestSignedByAnotherKey(
 	require.NoError(t, proto.Unmarshal(
 		signedManifest(t, files)[len(mfer.MAGIC):], outer))
 
+	// Armored blocks start on a line of their own; mfer, unlike gpg, ends
+	// one without a newline.
 	outer.SigningPubKey = slices.Concat(
-		required.GetSigningPubKey(), outer.GetSigningPubKey())
+		required.GetSigningPubKey(), []byte("\n"), outer.GetSigningPubKey())
 	outer.Signer = required.GetSigner()
 
 	data, err := proto.Marshal(outer)

internal/cli/export.go

@@ -36,7 +36,6 @@ func (mfa *CLIApp) exportManifestOperation(
 
 	defer func() { _ = rc.Close() }()
 
-	//nolint:contextcheck // mfer loads a manifest without a context
 	manifest, err := mfer.NewManifestFromReader(rc)
 	if err != nil {
 		return fmt.Errorf("parse manifest: %w", err)

internal/cli/fetch.go

@@ -455,7 +455,6 @@ func (mfa *CLIApp) fetchManifest(
 	}
 
 	// Parse manifest
-	//nolint:contextcheck // mfer loads a manifest without a context
 	manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
 	if err != nil {
 		return nil, nil, fmt.Errorf("parse manifest: %w", err)
@@ -463,7 +462,6 @@ func (mfa *CLIApp) fetchManifest(
 
 	requiredSigner := cmd.String(flagRequireSignature)
 	if requiredSigner != "" {
-		//nolint:contextcheck // mfer loads a manifest without a context
 		err = verifyFetchedSigner(manifestData, requiredSigner)
 		if err != nil {
 			return nil, nil, err

internal/cli/fetch_test.go

@@ -1191,20 +1191,23 @@ func TestFetchIntoDest(t *testing.T) {
 // with check's message before it downloads or writes anything; the
 // required key lets it through. A manifest signed by another key whose
 // embedded public key block also holds the required key must stop fetch
-// too. The signed cases need gpg and are skipped without it, as the other
-// signing tests are.
-//
-//nolint:paralleltest // signedManifest calls t.Setenv, which bars t.Parallel
+// too.
 func TestFetchRequireSignature(t *testing.T) {
+	t.Parallel()
+
 	files := map[string][]byte{testFileTxt: []byte("signed file")}
 
 	t.Run("unsigned", func(t *testing.T) {
+		t.Parallel()
+
 		assertFetchRefused(t, manifestOf(t, files), files,
 			"manifest is not signed, but signature from "+msgFpA+" is required",
 			"--"+flagRequireSignature, msgFpA)
 	})
 
 	t.Run("signed", func(t *testing.T) {
+		t.Parallel()
+
 		manifest := signedManifest(t, files)
 
 		signer := string(signedChecker(t, manifest).Signer())
@@ -1227,6 +1230,8 @@ func TestFetchRequireSignature(t *testing.T) {
 	})
 
 	t.Run("signed by another key embedded after the required one", func(t *testing.T) {
+		t.Parallel()
+
 		manifest, required := manifestSignedByAnotherKey(t, files)
 
 		assertFetchRefused(t, manifest, files,

internal/cli/freshen.go

@@ -339,7 +339,7 @@ func writeFreshenedManifest(
 
 // newFreshenBuilder constructs the manifest builder configured from CLI
 // flags.
-func newFreshenBuilder(cmd *cli.Command) *mfer.Builder {
+func (mfa *CLIApp) newFreshenBuilder(cmd *cli.Command) (*mfer.Builder, error) {
 	builder := mfer.NewBuilder()
 	if cmd.Bool("include-timestamps") {
 		builder.SetIncludeTimestamps(true)
@@ -347,13 +347,15 @@ func newFreshenBuilder(cmd *cli.Command) *mfer.Builder {
 
 	// Set up signing options if sign-key is provided
 	if signKey := cmd.String("sign-key"); signKey != "" {
-		builder.SetSigningOptions(&mfer.SigningOptions{
-			KeyID: mfer.GPGKeyID(signKey),
-		})
-		log.Infof("signing manifest with GPG key: %s", signKey)
+		signing, err := mfa.signingOptions(signKey)
+		if err != nil {
+			return nil, err
+		}
+
+		builder.SetSigningOptions(signing)
 	}
 
-	return builder
+	return builder, nil
 }
 
 // freshenScan runs the scan phase against the loaded manifest entries
@@ -433,7 +435,7 @@ func hashTotals(entries []*freshenEntry) (int64, int64) {
 }
 
 // runFreshenHash processes every entry through the hasher, aborting if
-// the context is canceled.
+// the context is canceled, and ends the hasher's progress line.
 func runFreshenHash(
 	ctx context.Context, hasher *freshenHasher, entries []*freshenEntry,
 ) error {
@@ -450,6 +452,10 @@ func runFreshenHash(
 		}
 	}
 
+	if hasher.showProgress && hasher.filesToHash > 0 {
+		log.ProgressDone()
+	}
+
 	return nil
 }
 
@@ -494,7 +500,11 @@ func (mfa *CLIApp) freshenManifestOperation(
 		return err
 	}
 
-	//nolint:contextcheck // mfer loads a manifest without a context
+	builder, err := mfa.newFreshenBuilder(cmd)
+	if err != nil {
+		return err
+	}
+
 	existingByPath, err := mfa.loadExistingEntries(manifestPath)
 	if err != nil {
 		return err
@@ -528,7 +538,7 @@ func (mfa *CLIApp) freshenManifestOperation(
 		totalHashBytes: totalHashBytes,
 		filesToHash:    filesToHash,
 		startHash:      time.Now(),
-		builder:        newFreshenBuilder(cmd),
+		builder:        builder,
 	}
 
 	err = runFreshenHash(ctx, hasher, scanner.entries)
@@ -536,10 +546,6 @@ func (mfa *CLIApp) freshenManifestOperation(
 		return err
 	}
 
-	if showProgress && filesToHash > 0 {
-		log.ProgressDone()
-	}
-
 	// Print summary
 	log.Infof("freshen complete: %d unchanged, %d changed, %d added, %d removed",
 		scanner.unchanged, scanner.changed, scanner.added, removed)

internal/cli/gen.go

@@ -124,7 +124,7 @@ func (mfa *CLIApp) outputPath(cmd *cli.Command) (string, error) {
 // the path the manifest is written to.
 func (mfa *CLIApp) buildScannerOptions(
 	cmd *cli.Command, output string,
-) *mfer.ScannerOptions {
+) (*mfer.ScannerOptions, error) {
 	opts := &mfer.ScannerOptions{
 		IncludeDotfiles:    cmd.Bool("include-dotfiles"),
 		FollowSymLinks:     cmd.Bool("follow-symlinks"),
@@ -145,13 +145,15 @@ func (mfa *CLIApp) buildScannerOptions(
 
 	// Set up signing options if sign-key is provided
 	if signKey := cmd.String("sign-key"); signKey != "" {
-		opts.SigningOptions = &mfer.SigningOptions{
-			KeyID: mfer.GPGKeyID(signKey),
+		signing, err := mfa.signingOptions(signKey)
+		if err != nil {
+			return nil, err
 		}
-		log.Infof("signing manifest with GPG key: %s", signKey)
+
+		opts.SigningOptions = signing
 	}
 
-	return opts
+	return opts, nil
 }
 
 // enumerateInputs runs the enumeration phase over the argument paths,
@@ -275,7 +277,12 @@ func (mfa *CLIApp) generateManifestOperation(
 		return err
 	}
 
-	s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(cmd, outputPath))
+	opts, err := mfa.buildScannerOptions(cmd, outputPath)
+	if err != nil {
+		return err
+	}
+
+	s := mfer.NewScannerWithOptions(opts)
 
 	// Phase 1: Enumeration - collect paths and stat files
 	err = mfa.runEnumeratePhase(cmd, s)

internal/cli/list.go

@@ -29,7 +29,6 @@ func (mfa *CLIApp) listManifestOperation(ctx context.Context, cmd *cli.Command)
 
 	defer func() { _ = rc.Close() }()
 
-	//nolint:contextcheck // mfer loads a manifest without a context
 	manifest, err := mfer.NewManifestFromReader(rc)
 	if err != nil {
 		return fmt.Errorf("parse manifest: %w", err)

internal/cli/mfer.go

@@ -169,7 +169,7 @@ func requireSignatureFlag() *cli.StringFlag {
 	return &cli.StringFlag{
 		Name:    flagRequireSignature,
 		Aliases: []string{"S"},
-		Usage:   "Require manifest to be signed by the specified GPG key ID",
+		Usage:   "Require manifest to be signed by the OpenPGP key with this fingerprint",
 		Sources: cli.EnvVars("MFER_REQUIRE_SIGNATURE"),
 	}
 }
@@ -229,7 +229,7 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
 			&cli.StringFlag{
 				Name:    "sign-key",
 				Aliases: []string{"s"},
-				Usage:   "GPG key ID to sign the manifest with",
+				Usage:   "OpenPGP secret key file to sign the manifest with",
 				Sources: cli.EnvVars("MFER_SIGN_KEY"),
 			},
 			&cli.StringFlag{
@@ -319,7 +319,7 @@ func (mfa *CLIApp) freshenCommand() *cli.Command {
 			&cli.StringFlag{
 				Name:    "sign-key",
 				Aliases: []string{"s"},
-				Usage:   "GPG key ID to sign the manifest with",
+				Usage:   "OpenPGP secret key file to sign the manifest with",
 				Sources: cli.EnvVars("MFER_SIGN_KEY"),
 			},
 			&cli.BoolFlag{

internal/cli/signing.go

@@ -0,0 +1,79 @@
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/spf13/afero"
+	"golang.org/x/term"
+	"sneak.berlin/go/mfer/internal/log"
+	"sneak.berlin/go/mfer/mfer"
+)
+
+// envSignKeyPassphrase names the environment variable holding the
+// passphrase of a protected signing key.
+//
+//nolint:gosec // G101: the name of a variable, not a credential
+const envSignKeyPassphrase = "MFER_SIGN_KEY_PASSPHRASE"
+
+// errNoPassphrase indicates a protected signing key whose passphrase is
+// neither in the environment nor can be asked for on a terminal.
+var errNoPassphrase = errors.New(
+	"signing key is protected: set " + envSignKeyPassphrase + " to its passphrase")
+
+// signingOptions returns the signing options for the OpenPGP secret key in
+// the file path. The passphrase of a protected key comes from
+// MFER_SIGN_KEY_PASSPHRASE, or else from the terminal on stdin.
+func (mfa *CLIApp) signingOptions(path string) (*mfer.SigningOptions, error) {
+	secretKey, err := afero.ReadFile(mfa.Fs, path)
+	if err != nil {
+		return nil, fmt.Errorf("read signing key: %w", err)
+	}
+
+	protected, err := mfer.SecretKeyIsProtected(secretKey)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", path, err)
+	}
+
+	log.Infof("signing manifest with the OpenPGP key in %s", path)
+
+	opts := &mfer.SigningOptions{SecretKey: secretKey}
+	if !protected {
+		return opts, nil
+	}
+
+	opts.Passphrase, err = mfa.readPassphrase(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return opts, nil
+}
+
+// readPassphrase returns MFER_SIGN_KEY_PASSPHRASE when it is set, or else
+// asks for the passphrase of the key in the file path on the terminal on
+// stdin.
+func (mfa *CLIApp) readPassphrase(path string) ([]byte, error) {
+	passphrase := os.Getenv(envSignKeyPassphrase)
+	if passphrase != "" {
+		return []byte(passphrase), nil
+	}
+
+	stdin, ok := mfa.Stdin.(*os.File)
+	if !ok || !term.IsTerminal(int(stdin.Fd())) {
+		return nil, errNoPassphrase
+	}
+
+	_, _ = fmt.Fprintf(mfa.Stderr, "Passphrase for %s: ", path)
+
+	typed, err := term.ReadPassword(int(stdin.Fd()))
+
+	_, _ = fmt.Fprintln(mfa.Stderr)
+
+	if err != nil {
+		return nil, fmt.Errorf("read passphrase: %w", err)
+	}
+
+	return typed, nil
+}

internal/cli/signing_test.go

@@ -0,0 +1,91 @@
+//nolint:testpackage // white-box tests exercise unexported internals
+package cli
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testFlagSignKey = "--sign-key"
+
+// TestGenAndFreshenSignWithKeyFile runs gen, then freshen after a file is
+// added, with --sign-key naming a key file: one key with no passphrase and
+// one protected by the passphrase in MFER_SIGN_KEY_PASSPHRASE. check
+// --require-signature must accept each manifest as signed by that key.
+// freshen leaves its manifest out of the listing only on the real
+// filesystem, so the test uses that.
+func TestGenAndFreshenSignWithKeyFile(t *testing.T) {
+	for name, passphrase := range map[string][]byte{
+		"unprotected": nil,
+		"protected":   []byte("passphrase"),
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Setenv(envSignKeyPassphrase, string(passphrase))
+
+			secretKey, fingerprint := testSecretKey(t, passphrase)
+
+			fs := afero.NewOsFs()
+			keyFile := filepath.Join(t.TempDir(), "key.asc")
+			root := t.TempDir()
+			manifestPath := filepath.Join(root, defaultManifestName)
+
+			require.NoError(t, afero.WriteFile(fs, keyFile, secretKey, 0o600))
+			writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
+
+			opts := testOpts([]string{
+				testApp, cmdGenerate, "-q", testFlagSignKey, keyFile,
+				"-o", manifestPath, root,
+			}, fs)
+			require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+
+			check := []string{
+				testApp, cmdCheck, "-q",
+				"--" + flagRequireSignature, fingerprint, manifestPath,
+			}
+
+			opts = testOpts(check, fs)
+			require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+
+			writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
+
+			opts = testOpts([]string{
+				testApp, cmdFreshen, "-q", testFlagSignKey, keyFile, manifestPath,
+			}, fs)
+			require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+
+			opts = testOpts(check, fs)
+			require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+			assert.Len(t, manifestFiles(t, fs, manifestPath), 2)
+		})
+	}
+}
+
+// TestSignWithProtectedKeyNeedsPassphrase runs gen with a protected key,
+// with MFER_SIGN_KEY_PASSPHRASE empty and no terminal to ask on. gen must
+// fail, naming the variable, and write no manifest.
+func TestSignWithProtectedKeyNeedsPassphrase(t *testing.T) {
+	t.Setenv(envSignKeyPassphrase, "")
+
+	secretKey, _ := testSecretKey(t, []byte("secret"))
+
+	fs := afero.NewMemMapFs()
+	require.NoError(t, afero.WriteFile(fs, "/key.asc", secretKey, 0o600))
+	require.NoError(t, fs.MkdirAll(testDir, 0o755))
+	writeTestFile(t, fs, testFile1, "hello")
+
+	opts := testOpts([]string{
+		testApp, cmdGenerate, "-q", testFlagSignKey, "/key.asc",
+		"-o", testMF, testDir,
+	}, fs)
+	assert.Equal(t, 1, runCLI(opts))
+	assert.Contains(t, testStderr(t, opts),
+		"signing key is protected: set MFER_SIGN_KEY_PASSPHRASE to its passphrase")
+
+	exists, err := afero.Exists(fs, testMF)
+	require.NoError(t, err)
+	assert.False(t, exists)
+}