Stop vault creation when a vault's metadata cannot be read (closes #140)

GetNextDerivationIndex passed over a vault whose vault-metadata.json could
not be read or parsed, so the next vault made from the same mnemonic could
get that vault's derivation index and so the same long-term key. It now
returns an error naming the file, so vault create and init stop with that
error and leave the state directory as it was. A vault directory with no
metadata file is still skipped.

The regression test breaks one vault's metadata, once with content that
does not parse and once with a test filesystem that fails to open it, and
requires CreateVault with the same mnemonic to fail, naming the file, and
leave no new vault.

Model: opus-5-5

README.md

@@ -313,11 +313,6 @@ generates a new Age key.
 
 Decrypts data using an Age key stored as a secret.
 
-With `--output`, both hold the result in memory and write it to `--output` only
-once encryption or decryption has succeeded, so a failure to encrypt or decrypt
-leaves `--output` untouched. A failure while writing `--output`, such as a full
-disk, can leave it empty or partly written.
-
 ## Storage Architecture
 
 ### Directory Structure
@@ -555,7 +550,11 @@ secret decrypt encryption/mykey --input document.txt.age --output document.txt
 ### Vault Management
 
 - **Derivation Index**: Each vault uses a unique derivation index from the
-  mnemonic, and thus a unique key pair
+  mnemonic, and thus a unique key pair; creating a vault (`secret init` and
+  `secret vault create`) fails, with an error naming the file, when any vault's
+  `vault-metadata.json` exists but cannot be read or parsed; a vault directory
+  with no `vault-metadata.json` is skipped, so the index it uses can be given to
+  a new vault
 - **Public Key Hash**: Double SHA-256 hash of the vault's public key; the same
   hash of the index-0 public key identifies vaults from the same mnemonic
 - **Automatic Key Derivation**: When creating vaults with a mnemonic, keys are

internal/cli/crypto.go

@@ -1,7 +1,6 @@
 package cli
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -108,8 +107,8 @@ func (cli *Instance) storeNewEncryptionKey(
 // does not exist. The caller must destroy the returned buffer. Only storing
 // a new key takes the state directory lock, so that reading an existing key
 // works on a read-only state directory and keeps no other command waiting
-// at the passphrase prompt, and Encrypt reads its input and writes its
-// output without the lock.
+// at the passphrase prompt, and Encrypt streams its input and output
+// unlocked.
 func (cli *Instance) resolveEncryptionKey(
 	vlt *vault.Vault, secretName string,
 ) (*memguard.LockedBuffer, error) {
@@ -189,17 +188,17 @@ func (cli *Instance) Encrypt(secretName, inputFile, outputFile string) error {
 		input = file
 	}
 
-	// Set up output writer. With --output the result is held in memory
-	// and written only once encryption has succeeded, so a failure to
-	// encrypt leaves --output untouched and --input can be the same file.
-	// A failure while writing --output, such as a full disk, can leave it
-	// empty or partly written.
+	// Set up output writer
 	output := cli.cmd.OutOrStdout()
 
-	var result bytes.Buffer
-
 	if outputFile != "" {
-		output = &result
+		file, err := cli.fs.Create(outputFile)
+		if err != nil {
+			return fmt.Errorf("failed to create output file: %w", err)
+		}
+		defer func() { _ = file.Close() }()
+
+		output = file
 	}
 
 	// Encrypt the data
@@ -218,11 +217,7 @@ func (cli *Instance) Encrypt(secretName, inputFile, outputFile string) error {
 		return fmt.Errorf("failed to finalize encryption: %w", err)
 	}
 
-	if outputFile == "" {
-		return nil
-	}
-
-	return cli.writeOutputFile(outputFile, result.Bytes())
+	return nil
 }
 
 // Decrypt decrypts data using an age secret key stored in a secret
@@ -283,17 +278,17 @@ func (cli *Instance) Decrypt(secretName, inputFile, outputFile string) error {
 		input = file
 	}
 
-	// Set up output writer. With --output the result is held in memory
-	// and written only once decryption has succeeded, so a failure to
-	// decrypt leaves --output untouched and --input can be the same file.
-	// A failure while writing --output, such as a full disk, can leave it
-	// empty or partly written.
+	// Set up output writer
 	output := cli.cmd.OutOrStdout()
 
-	var result bytes.Buffer
-
 	if outputFile != "" {
-		output = &result
+		file, err := cli.fs.Create(outputFile)
+		if err != nil {
+			return fmt.Errorf("failed to create output file: %w", err)
+		}
+		defer func() { _ = file.Close() }()
+
+		output = file
 	}
 
 	// Decrypt the data
@@ -307,32 +302,6 @@ func (cli *Instance) Decrypt(secretName, inputFile, outputFile string) error {
 		return fmt.Errorf("failed to decrypt data: %w", err)
 	}
 
-	if outputFile == "" {
-		return nil
-	}
-
-	return cli.writeOutputFile(outputFile, result.Bytes())
-}
-
-// writeOutputFile creates outputFile, or empties it if it exists, and
-// writes result into it.
-func (cli *Instance) writeOutputFile(outputFile string, result []byte) error {
-	file, err := cli.fs.Create(outputFile)
-	if err != nil {
-		return fmt.Errorf("failed to create output file: %w", err)
-	}
-
-	_, err = file.Write(result)
-
-	closeErr := file.Close()
-	if err == nil {
-		err = closeErr
-	}
-
-	if err != nil {
-		return fmt.Errorf("failed to write output file: %w", err)
-	}
-
 	return nil
 }
 

internal/cli/crypto_test.go

@@ -1,185 +0,0 @@
-//nolint:testpackage // sets the unexported fields of Instance
-package cli
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/spf13/cobra"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/secret/internal/vault"
-)
-
-// plainText is the content of plain.txt in the directory newCryptoTest makes.
-const plainText = "document"
-
-// newCryptoTest returns an Instance whose current vault, in a new state
-// directory on the real filesystem, holds the age key secret "key", and a
-// new directory holding only plain.txt and plain.txt.age, its encryption
-// with that key.
-func newCryptoTest(t *testing.T) (*Instance, string) {
-	t.Helper()
-
-	fs := afero.NewOsFs()
-	stateDir := t.TempDir()
-
-	_, err := vault.CreateVault(fs, stateDir, "default", testMnemonicBuffer(t), nil)
-	require.NoError(t, err)
-
-	cli := NewCLIInstanceWithStateDir(fs, stateDir)
-	cli.Mnemonic = testMnemonicBuffer(t)
-	cli.cmd = &cobra.Command{}
-
-	dir := t.TempDir()
-	plain := filepath.Join(dir, "plain.txt")
-	require.NoError(t, afero.WriteFile(fs, plain, []byte(plainText), 0o600))
-	require.NoError(t, cli.Encrypt("key", plain, plain+".age"))
-
-	return cli, dir
-}
-
-// TestFailedCryptoLeavesOutputUnchanged is a regression test for
-// https://git.eeqj.de/sneak/secret/issues/143, where secret encrypt and
-// secret decrypt emptied the file named by --output before they failed.
-// A failed command must leave an existing file as it was and create none,
-// and give the same error as without --output.
-func TestFailedCryptoLeavesOutputUnchanged(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name string
-		run  func(cli *Instance, dir, output string) error
-	}{
-		// Reading a directory fails once encryption has begun
-		{"encrypt", func(cli *Instance, dir, output string) error {
-			return cli.Encrypt("key", dir, output)
-		}},
-		// plain.txt is not an age file
-		{"decrypt", func(cli *Instance, dir, output string) error {
-			return cli.Decrypt("key", filepath.Join(dir, "plain.txt"), output)
-		}},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			cli, dir := newCryptoTest(t)
-			existing := filepath.Join(dir, "existing.txt")
-			require.NoError(t, afero.WriteFile(cli.fs, existing, []byte("precious"), 0o600))
-
-			cli.cmd.SetOut(io.Discard)
-			toStdout := tt.run(cli, dir, "")
-			require.Error(t, toStdout)
-
-			require.EqualError(t, tt.run(cli, dir, existing), toStdout.Error())
-			require.EqualError(t, tt.run(cli, dir, filepath.Join(dir, "new.txt")),
-				toStdout.Error())
-
-			content, err := afero.ReadFile(cli.fs, existing)
-			require.NoError(t, err)
-			require.Equal(t, "precious", string(content))
-
-			exists, err := afero.Exists(cli.fs, filepath.Join(dir, "new.txt"))
-			require.NoError(t, err)
-			require.False(t, exists, "a failed %s created --output", tt.name)
-		})
-	}
-}
-
-// TestCryptoInPlace checks that secret encrypt and secret decrypt given
-// one file as both --input and --output replace it with the right result.
-// Before https://git.eeqj.de/sneak/secret/issues/143 they emptied the file
-// before reading it.
-func TestCryptoInPlace(t *testing.T) {
-	t.Parallel()
-
-	cli, dir := newCryptoTest(t)
-	file := filepath.Join(dir, "plain.txt")
-
-	require.NoError(t, cli.Encrypt("key", file, file))
-	require.NoError(t, cli.Decrypt("key", file, file))
-
-	content, err := afero.ReadFile(cli.fs, file)
-	require.NoError(t, err)
-	require.Equal(t, plainText, string(content))
-}
-
-// errCloseFailed is returned by the files closeFailFs creates in place of
-// a successful Close, as NFS reports a full disk only when the file is
-// closed.
-var errCloseFailed = errors.New("no space left on device")
-
-// closeFailFs creates files whose Close fails with errCloseFailed.
-type closeFailFs struct {
-	afero.Fs
-}
-
-// closeFailFile is a file created through closeFailFs.
-type closeFailFile struct {
-	afero.File
-}
-
-func (f closeFailFile) Close() error {
-	_ = f.File.Close()
-
-	return errCloseFailed
-}
-
-//nolint:ireturn // afero.File is the interface required by afero.Fs
-func (f closeFailFs) Create(name string) (afero.File, error) {
-	file, err := f.Fs.Create(name)
-	if err != nil {
-		return nil, err
-	}
-
-	return closeFailFile{File: file}, nil
-}
-
-// TestDecryptFailsWhenClosingOutputFails checks that secret decrypt fails
-// when closing --output fails, since some filesystems report a failed
-// write only then.
-func TestDecryptFailsWhenClosingOutputFails(t *testing.T) {
-	t.Parallel()
-
-	cli, dir := newCryptoTest(t)
-	cli.fs = closeFailFs{Fs: cli.fs}
-
-	err := cli.Decrypt("key", filepath.Join(dir, "plain.txt.age"),
-		filepath.Join(dir, "out.txt"))
-	require.ErrorIs(t, err, errCloseFailed)
-}
-
-// TestDecryptToDescriptorOfDeletedFile checks that secret decrypt given
-// /dev/fd/N as --output writes through that descriptor. A program may give
-// the command a temporary file it has already deleted as stdout, pass
-// /dev/stdout as --output and read the result back from that file.
-func TestDecryptToDescriptorOfDeletedFile(t *testing.T) {
-	t.Parallel()
-
-	cli, dir := newCryptoTest(t)
-
-	file, err := os.CreateTemp(t.TempDir(), "out")
-	require.NoError(t, err)
-
-	defer func() { _ = file.Close() }()
-
-	require.NoError(t, os.Remove(file.Name()))
-
-	output := fmt.Sprintf("/dev/fd/%d", file.Fd())
-	require.NoError(t, cli.Decrypt("key", filepath.Join(dir, "plain.txt.age"), output))
-
-	// On macOS the command's open of /dev/fd/N shares this descriptor's
-	// position, which its write leaves at the end.
-	_, err = file.Seek(0, io.SeekStart)
-	require.NoError(t, err)
-
-	content, err := io.ReadAll(file)
-	require.NoError(t, err)
-	require.Equal(t, plainText, string(content))
-}

internal/vault/metadata.go

@@ -4,7 +4,9 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"github.com/spf13/afero"
@@ -33,7 +35,10 @@ func ComputeDoubleSHA256(data []byte) string {
 }
 
 // GetNextDerivationIndex finds the next available derivation index for a given mnemonic
-// by deriving the public key for index 0 and using its hash to identify related vaults
+// by deriving the public key for index 0 and using its hash to identify related vaults.
+// It fails when a vault's metadata file exists but cannot be read or parsed: the
+// index that vault uses is then unknown, and handing it out again would give a
+// second vault the same long-term key.
 func GetNextDerivationIndex(
 	fs afero.Fs, stateDir string, mnemonic string,
 ) (uint32, error) {
@@ -72,21 +77,23 @@ func GetNextDerivationIndex(
 			continue
 		}
 
-		// Try to read vault metadata
 		metadataPath := filepath.Join(vaultsDir, entry.Name(), "vault-metadata.json")
 
 		metadataBytes, err := afero.ReadFile(fs, metadataPath)
-		if err != nil {
-			// Skip vaults without metadata
+		if errors.Is(err, os.ErrNotExist) {
+			// Skip directories without metadata
 			continue
 		}
 
+		if err != nil {
+			return 0, fmt.Errorf("failed to read vault metadata %s: %w", metadataPath, err)
+		}
+
 		var metadata Metadata
 
 		err = json.Unmarshal(metadataBytes, &metadata)
 		if err != nil {
-			// Skip vaults with invalid metadata
-			continue
+			return 0, fmt.Errorf("failed to parse vault metadata %s: %w", metadataPath, err)
 		}
 
 		// Check if this vault uses the same mnemonic by comparing family hashes

internal/vault/metadata_test.go

@@ -1,11 +1,15 @@
 package vault_test
 
 import (
+	"errors"
 	"path/filepath"
 	"strings"
 	"testing"
 
 	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/secret/internal/secret"
 	"sneak.berlin/go/secret/internal/vault"
 	"sneak.berlin/go/secret/pkg/agehd"
 )
@@ -163,6 +167,88 @@ func testGetNextDerivationIndex(t *testing.T, fs afero.Fs) {
 	}
 }
 
+// errMetadataUnreadable is returned by the test filesystem in place of a
+// successful open of one vault's metadata file.
+var errMetadataUnreadable = errors.New("input/output error")
+
+// metadataReadFailFs fails every open of the file at unreadablePath. The
+// file still exists, so checking for it succeeds and only reading it fails.
+type metadataReadFailFs struct {
+	afero.Fs
+
+	unreadablePath string
+}
+
+//nolint:ireturn // afero.File is the interface required by afero.Fs
+func (f *metadataReadFailFs) Open(name string) (afero.File, error) {
+	if name == f.unreadablePath {
+		return nil, errMetadataUnreadable
+	}
+
+	return f.Fs.Open(name)
+}
+
+// TestCreateVaultWithCorruptMetadataFails checks that creating a vault from
+// a mnemonic fails, naming the file and leaving no new vault, when another
+// vault's metadata file exists but cannot be parsed or read. That vault's
+// derivation index is then unknown, so the new vault could get the same one
+// and the same key.
+func TestCreateVaultWithCorruptMetadataFails(t *testing.T) {
+	t.Parallel()
+
+	vaultsDir := filepath.Join(testStateDir, "vaults.d")
+	metadataPath := filepath.Join(vaultsDir, "work", "vault-metadata.json")
+
+	tests := []struct {
+		name string
+		// breakMetadata makes the work vault's metadata file unusable on
+		// base and returns the filesystem to create the second vault on.
+		breakMetadata func(t *testing.T, base afero.Fs) afero.Fs
+	}{
+		{
+			name: "parsing the file fails",
+			breakMetadata: func(t *testing.T, base afero.Fs) afero.Fs {
+				t.Helper()
+
+				err := afero.WriteFile(base, metadataPath, []byte("{"),
+					secret.FilePerms)
+				require.NoError(t, err)
+
+				return base
+			},
+		},
+		{
+			name: "reading the file fails",
+			breakMetadata: func(_ *testing.T, base afero.Fs) afero.Fs {
+				return &metadataReadFailFs{Fs: base, unreadablePath: metadataPath}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			base := afero.NewMemMapFs()
+
+			_, err := vault.CreateVault(base, testStateDir, "work",
+				testMnemonicBuffer(t), nil)
+			require.NoError(t, err)
+
+			fs := tt.breakMetadata(t, base)
+
+			_, err = vault.CreateVault(fs, testStateDir, "other",
+				testMnemonicBuffer(t), nil)
+			require.ErrorContains(t, err, metadataPath)
+
+			entries, err := afero.ReadDir(base, vaultsDir)
+			require.NoError(t, err)
+			require.Len(t, entries, 1)
+			assert.Equal(t, "work", entries[0].Name())
+		})
+	}
+}
+
 func testMetadataPersistence(t *testing.T, fs afero.Fs) {
 	t.Helper()