Hold the encrypt and decrypt result in memory until it is written (closes #143)

With --output, secret encrypt and secret decrypt emptied that file before
reading any input, so a failed command left it empty, and encrypting a file
onto itself gave the encryption of nothing.

With --output they now hold the result in memory and open --output, with the
same call as before, only once encryption or decryption has succeeded. A
failure to encrypt or decrypt leaves --output untouched, and --input can be the
same file. A failure while writing --output, such as a full disk, can leave it
empty or partly written; an error from closing it is reported. Without
--output the result still goes to stdout as it is produced.

Model: opus-5-5

README.md

@@ -129,6 +129,17 @@ one.
 
 ### Secret Management
 
+#### Secret Size
+
+While `secret` works on a secret's value it keeps the value in locked memory, so
+the value counts against the locked-memory limit (`ulimit -l`). `secret` raises
+that limit to the hard limit when it starts. `secret add`, `secret import` and
+`secret generate secret` refuse a value that does not fit in the limit with 8
+memory pages to spare (32 KiB with 4 KiB pages), with an error that names the
+limit. Linux's usual limit of 8 MiB allows a value of nearly 8 MiB; a larger one
+needs a higher hard limit, such as `memlock` in `/etc/security/limits.conf`. No
+value may be over 100 MiB, whatever the limit.
+
 #### `secret add <secret-name> [--force]`
 
 Adds a secret to the current vault. Reads the secret value from stdin.
@@ -193,6 +204,12 @@ Moves or renames a secret within the current vault.
 Lists all versions of a secret showing creation time, status, and validity
 period.
 
+`STATUS` is `current` for the secret's current version, `expired` for another
+version whose `NOT_AFTER` has passed, and `inactive` for any other version.
+Adding a version sets `NOT_AFTER` on the version that was current;
+`secret version promote` sets no times. A version that is not current therefore
+shows `inactive` when no version was added while it was current.
+
 #### `secret version promote <secret-name> <version>`
 
 Promotes a specific version to current by rewriting the secret's `current` file
@@ -296,6 +313,11 @@ generates a new Age key.
 
 Decrypts data using an Age key stored as a secret.
 
+With `--output`, both hold the result in memory and write it to `--output` only
+once encryption or decryption has succeeded, so a failure to encrypt or decrypt
+leaves `--output` untouched. A failure while writing `--output`, such as a full
+disk, can leave it empty or partly written.
+
 ## Storage Architecture
 
 ### Directory Structure

internal/cli/crypto.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -107,8 +108,8 @@ func (cli *Instance) storeNewEncryptionKey(
 // does not exist. The caller must destroy the returned buffer. Only storing
 // a new key takes the state directory lock, so that reading an existing key
 // works on a read-only state directory and keeps no other command waiting
-// at the passphrase prompt, and Encrypt streams its input and output
+// at the passphrase prompt, and Encrypt reads its input and writes its
-// unlocked.
+// output without the lock.
 func (cli *Instance) resolveEncryptionKey(
 	vlt *vault.Vault, secretName string,
 ) (*memguard.LockedBuffer, error) {
@@ -188,17 +189,17 @@ func (cli *Instance) Encrypt(secretName, inputFile, outputFile string) error {
 		input = file
 	}
 
-	// Set up output writer
+	// Set up output writer. With --output the result is held in memory
+	// and written only once encryption has succeeded, so a failure to
+	// encrypt leaves --output untouched and --input can be the same file.
+	// A failure while writing --output, such as a full disk, can leave it
+	// empty or partly written.
 	output := cli.cmd.OutOrStdout()
 
-	if outputFile != "" {
+	var result bytes.Buffer
-		file, err := cli.fs.Create(outputFile)
-		if err != nil {
-			return fmt.Errorf("failed to create output file: %w", err)
-		}
-		defer func() { _ = file.Close() }()
 
-		output = file
+	if outputFile != "" {
+		output = &result
 	}
 
 	// Encrypt the data
@@ -217,9 +218,13 @@ func (cli *Instance) Encrypt(secretName, inputFile, outputFile string) error {
 		return fmt.Errorf("failed to finalize encryption: %w", err)
 	}
 
+	if outputFile == "" {
 		return nil
 	}
 
+	return cli.writeOutputFile(outputFile, result.Bytes())
+}
+
 // Decrypt decrypts data using an age secret key stored in a secret
 func (cli *Instance) Decrypt(secretName, inputFile, outputFile string) error {
 	err := vault.ValidateSecretName(secretName)
@@ -278,17 +283,17 @@ func (cli *Instance) Decrypt(secretName, inputFile, outputFile string) error {
 		input = file
 	}
 
-	// Set up output writer
+	// Set up output writer. With --output the result is held in memory
+	// and written only once decryption has succeeded, so a failure to
+	// decrypt leaves --output untouched and --input can be the same file.
+	// A failure while writing --output, such as a full disk, can leave it
+	// empty or partly written.
 	output := cli.cmd.OutOrStdout()
 
-	if outputFile != "" {
+	var result bytes.Buffer
-		file, err := cli.fs.Create(outputFile)
-		if err != nil {
-			return fmt.Errorf("failed to create output file: %w", err)
-		}
-		defer func() { _ = file.Close() }()
 
-		output = file
+	if outputFile != "" {
+		output = &result
 	}
 
 	// Decrypt the data
@@ -302,6 +307,32 @@ func (cli *Instance) Decrypt(secretName, inputFile, outputFile string) error {
 		return fmt.Errorf("failed to decrypt data: %w", err)
 	}
 
+	if outputFile == "" {
+		return nil
+	}
+
+	return cli.writeOutputFile(outputFile, result.Bytes())
+}
+
+// writeOutputFile creates outputFile, or empties it if it exists, and
+// writes result into it.
+func (cli *Instance) writeOutputFile(outputFile string, result []byte) error {
+	file, err := cli.fs.Create(outputFile)
+	if err != nil {
+		return fmt.Errorf("failed to create output file: %w", err)
+	}
+
+	_, err = file.Write(result)
+
+	closeErr := file.Close()
+	if err == nil {
+		err = closeErr
+	}
+
+	if err != nil {
+		return fmt.Errorf("failed to write output file: %w", err)
+	}
+
 	return nil
 }
 

internal/cli/crypto_test.go

@@ -0,0 +1,185 @@
+//nolint:testpackage // sets the unexported fields of Instance
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/secret/internal/vault"
+)
+
+// plainText is the content of plain.txt in the directory newCryptoTest makes.
+const plainText = "document"
+
+// newCryptoTest returns an Instance whose current vault, in a new state
+// directory on the real filesystem, holds the age key secret "key", and a
+// new directory holding only plain.txt and plain.txt.age, its encryption
+// with that key.
+func newCryptoTest(t *testing.T) (*Instance, string) {
+	t.Helper()
+
+	fs := afero.NewOsFs()
+	stateDir := t.TempDir()
+
+	_, err := vault.CreateVault(fs, stateDir, "default", testMnemonicBuffer(t), nil)
+	require.NoError(t, err)
+
+	cli := NewCLIInstanceWithStateDir(fs, stateDir)
+	cli.Mnemonic = testMnemonicBuffer(t)
+	cli.cmd = &cobra.Command{}
+
+	dir := t.TempDir()
+	plain := filepath.Join(dir, "plain.txt")
+	require.NoError(t, afero.WriteFile(fs, plain, []byte(plainText), 0o600))
+	require.NoError(t, cli.Encrypt("key", plain, plain+".age"))
+
+	return cli, dir
+}
+
+// TestFailedCryptoLeavesOutputUnchanged is a regression test for
+// https://git.eeqj.de/sneak/secret/issues/143, where secret encrypt and
+// secret decrypt emptied the file named by --output before they failed.
+// A failed command must leave an existing file as it was and create none,
+// and give the same error as without --output.
+func TestFailedCryptoLeavesOutputUnchanged(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		run  func(cli *Instance, dir, output string) error
+	}{
+		// Reading a directory fails once encryption has begun
+		{"encrypt", func(cli *Instance, dir, output string) error {
+			return cli.Encrypt("key", dir, output)
+		}},
+		// plain.txt is not an age file
+		{"decrypt", func(cli *Instance, dir, output string) error {
+			return cli.Decrypt("key", filepath.Join(dir, "plain.txt"), output)
+		}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			cli, dir := newCryptoTest(t)
+			existing := filepath.Join(dir, "existing.txt")
+			require.NoError(t, afero.WriteFile(cli.fs, existing, []byte("precious"), 0o600))
+
+			cli.cmd.SetOut(io.Discard)
+			toStdout := tt.run(cli, dir, "")
+			require.Error(t, toStdout)
+
+			require.EqualError(t, tt.run(cli, dir, existing), toStdout.Error())
+			require.EqualError(t, tt.run(cli, dir, filepath.Join(dir, "new.txt")),
+				toStdout.Error())
+
+			content, err := afero.ReadFile(cli.fs, existing)
+			require.NoError(t, err)
+			require.Equal(t, "precious", string(content))
+
+			exists, err := afero.Exists(cli.fs, filepath.Join(dir, "new.txt"))
+			require.NoError(t, err)
+			require.False(t, exists, "a failed %s created --output", tt.name)
+		})
+	}
+}
+
+// TestCryptoInPlace checks that secret encrypt and secret decrypt given
+// one file as both --input and --output replace it with the right result.
+// Before https://git.eeqj.de/sneak/secret/issues/143 they emptied the file
+// before reading it.
+func TestCryptoInPlace(t *testing.T) {
+	t.Parallel()
+
+	cli, dir := newCryptoTest(t)
+	file := filepath.Join(dir, "plain.txt")
+
+	require.NoError(t, cli.Encrypt("key", file, file))
+	require.NoError(t, cli.Decrypt("key", file, file))
+
+	content, err := afero.ReadFile(cli.fs, file)
+	require.NoError(t, err)
+	require.Equal(t, plainText, string(content))
+}
+
+// errCloseFailed is returned by the files closeFailFs creates in place of
+// a successful Close, as NFS reports a full disk only when the file is
+// closed.
+var errCloseFailed = errors.New("no space left on device")
+
+// closeFailFs creates files whose Close fails with errCloseFailed.
+type closeFailFs struct {
+	afero.Fs
+}
+
+// closeFailFile is a file created through closeFailFs.
+type closeFailFile struct {
+	afero.File
+}
+
+func (f closeFailFile) Close() error {
+	_ = f.File.Close()
+
+	return errCloseFailed
+}
+
+//nolint:ireturn // afero.File is the interface required by afero.Fs
+func (f closeFailFs) Create(name string) (afero.File, error) {
+	file, err := f.Fs.Create(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return closeFailFile{File: file}, nil
+}
+
+// TestDecryptFailsWhenClosingOutputFails checks that secret decrypt fails
+// when closing --output fails, since some filesystems report a failed
+// write only then.
+func TestDecryptFailsWhenClosingOutputFails(t *testing.T) {
+	t.Parallel()
+
+	cli, dir := newCryptoTest(t)
+	cli.fs = closeFailFs{Fs: cli.fs}
+
+	err := cli.Decrypt("key", filepath.Join(dir, "plain.txt.age"),
+		filepath.Join(dir, "out.txt"))
+	require.ErrorIs(t, err, errCloseFailed)
+}
+
+// TestDecryptToDescriptorOfDeletedFile checks that secret decrypt given
+// /dev/fd/N as --output writes through that descriptor. A program may give
+// the command a temporary file it has already deleted as stdout, pass
+// /dev/stdout as --output and read the result back from that file.
+func TestDecryptToDescriptorOfDeletedFile(t *testing.T) {
+	t.Parallel()
+
+	cli, dir := newCryptoTest(t)
+
+	file, err := os.CreateTemp(t.TempDir(), "out")
+	require.NoError(t, err)
+
+	defer func() { _ = file.Close() }()
+
+	require.NoError(t, os.Remove(file.Name()))
+
+	output := fmt.Sprintf("/dev/fd/%d", file.Fd())
+	require.NoError(t, cli.Decrypt("key", filepath.Join(dir, "plain.txt.age"), output))
+
+	// On macOS the command's open of /dev/fd/N shares this descriptor's
+	// position, which its write leaves at the end.
+	_, err = file.Seek(0, io.SeekStart)
+	require.NoError(t, err)
+
+	content, err := io.ReadAll(file)
+	require.NoError(t, err)
+	require.Equal(t, plainText, string(content))
+}

internal/cli/generate.go

@@ -72,6 +72,12 @@ func newGenerateSecretCmd() *cobra.Command {
 				return fmt.Errorf("failed to initialize CLI: %w", err)
 			}
 
+			// Before readSecretEnv, which makes the first locked allocation
+			err = checkLockedMemoryLimit(0)
+			if err != nil {
+				return err
+			}
+
 			destroySecrets := cli.readSecretEnv()
 			defer destroySecrets()
 
@@ -135,10 +141,16 @@ func (cli *Instance) GenerateSecret(
 		return errLengthTooSmall
 	}
 
-	var (
+	if length > maxSecretSize {
-		secretValue string
+		return errSecretTooLarge
-		err         error
+	}
-	)
+
+	err := checkLockedMemoryLimit(length)
+	if err != nil {
+		return err
+	}
+
+	var secretValue string
 
 	switch secretType {
 	case "base58":

internal/cli/root.go

@@ -14,6 +14,8 @@ import (
 // every memguard buffer before it returns, so the caller must do nothing
 // but exit with the code.
 func Entry() int {
+	raiseLockedMemoryLimit()
+
 	// On SIGINT or SIGTERM memguard runs this function, wipes every buffer
 	// and exits with status 1. The passphrase prompt turns echo off on the
 	// controlling terminal until the read finishes, so a signal there would
@@ -51,6 +53,19 @@ func Entry() int {
 	return 0
 }
 
+// raiseLockedMemoryLimit raises this process's limit on locked memory, which
+// holds every secret value, to the hard limit, the most a process may set
+// without privileges. When that fails the limit stays as it was, and
+// checkLockedMemoryLimit names it in its error.
+func raiseLockedMemoryLimit() {
+	var limit unix.Rlimit
+
+	if unix.Getrlimit(unix.RLIMIT_MEMLOCK, &limit) == nil {
+		limit.Cur = limit.Max
+		_ = unix.Setrlimit(unix.RLIMIT_MEMLOCK, &limit)
+	}
+}
+
 func newRootCmd() *cobra.Command {
 	secret.Debug("newRootCmd starting")
 

internal/cli/secrets.go

@@ -14,6 +14,7 @@ import (
 	"github.com/awnumar/memguard"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
+	"golang.org/x/sys/unix"
 	"sneak.berlin/go/secret/internal/secret"
 	"sneak.berlin/go/secret/internal/vault"
 )
@@ -24,26 +25,30 @@ const (
 	// vaultSecretParts is the number of parts when splitting vault:secret
 	vaultSecretParts = 2
 
-	// initialBufferSize is the starting size for secret read buffers (4KB)
-	initialBufferSize = 4 * 1024
 	// maxSecretSize is the maximum allowed size of a secret (100MB)
 	maxSecretSize = 100 * 1024 * 1024
+
+	// lockedMemoryReservePages is the locked memory, in pages, that `secret
+	// add`, `secret import` and `secret generate secret` may need besides
+	// the value they store, for keys and the read buffer: memguard locks
+	// whole pages, at least one for each buffer. `secret add` was measured
+	// to need 7 with both SB_SECRET_MNEMONIC and SB_UNLOCK_PASSPHRASE set,
+	// and 6 with one; the eighth is to spare.
+	lockedMemoryReservePages = 8
+	// kibibyte is the unit of the locked-memory limit in `ulimit -l`.
+	kibibyte = 1024
 )
 
 // Sentinel errors for secret operations
 var (
 	errSecretTooLarge    = errors.New("secret too large: exceeds 100MB limit")
+	errLockedMemoryLimit = errors.New(
+		"secret too large for the locked-memory limit")
 	errCrossVaultSourceUnqualified = errors.New(
 		"source must specify vault (e.g., vault:secret) for cross-vault move")
 	errMoveOntoItself = errors.New("cannot be moved onto itself")
 )
 
-// bufferInfo tracks a protected buffer and the number of bytes used in it
-type bufferInfo struct {
-	buffer *memguard.LockedBuffer
-	used   int
-}
-
 // ParseVaultSecretRef parses a "vault:secret" or just "secret" reference
 // Returns (vaultName, secretName, isQualified)
 // If no vault is specified, returns empty vaultName and isQualified=false
@@ -75,6 +80,12 @@ func newAddCmd() *cobra.Command {
 
 			cli.cmd = cmd // Set the command for stdin access
 
+			// Before readSecretEnv, which makes the first locked allocation
+			err = checkLockedMemoryLimit(0)
+			if err != nil {
+				return err
+			}
+
 			destroySecrets := cli.readSecretEnv()
 			defer destroySecrets()
 
@@ -174,6 +185,12 @@ func newImportCmd() *cobra.Command {
 				return fmt.Errorf("failed to initialize CLI: %w", err)
 			}
 
+			// Before readSecretEnv, which makes the first locked allocation
+			err = checkLockedMemoryLimit(0)
+			if err != nil {
+				return err
+			}
+
 			destroySecrets := cli.readSecretEnv()
 			defer destroySecrets()
 
@@ -270,88 +287,105 @@ The source secret is deleted after successful copy.`,
 	return cmd
 }
 
-// updateBufferSize updates the buffer size based on usage pattern
+// checkLockedMemoryLimit returns an error naming the locked-memory limit
-func updateBufferSize(currentSize int, sameSize *int) int {
+// (`ulimit -l`) when a value of size bytes does not fit in it with
-	const (
+// lockedMemoryReservePages. memguard, which holds secret values in locked
-		doubleAfterBuffers = 2
+// memory, crashes or hangs the process when it cannot lock more, so a
-		growthFactor       = 2
+// command calls this with size 0 before its first locked allocation, and
-	)
+// again with the value's size before a buffer for the value is allocated.
+func checkLockedMemoryLimit(size int) error {
+	var limit unix.Rlimit
 
-	*sameSize++
+	err := unix.Getrlimit(unix.RLIMIT_MEMLOCK, &limit)
-	if *sameSize >= doubleAfterBuffers {
+	if err != nil {
-		*sameSize = 0
+		return fmt.Errorf("failed to read the locked-memory limit: %w", err)
-
-		return currentSize * growthFactor
 	}
 
-	return currentSize
+	// memguard locks whole pages, so the value is counted in whole pages.
+	pageSize := os.Getpagesize()
+	valuePages := (size + pageSize - 1) / pageSize
+
+	//nolint:gosec // G115: a page count and a page size are never negative
+	needed := uint64((valuePages + lockedMemoryReservePages) * pageSize)
+	if needed > limit.Cur {
+		return fmt.Errorf("%w of %d KiB (ulimit -l)",
+			errLockedMemoryLimit, limit.Cur/kibibyte)
 	}
 
-// destroyBuffers destroys every buffer in the list
+	return nil
-func destroyBuffers(buffers []bufferInfo) {
-	for _, b := range buffers {
-		b.buffer.Destroy()
-	}
 }
 
-// readSecretFromReader reads all data from reader into protected buffers,
+// readSecretFromReader reads all of reader into a new locked buffer, which
-// enforcing the maximum secret size. On failure the accumulated buffers
+// the caller must destroy, leaving out one trailing newline when trimNewline
-// are destroyed; on success the caller must destroy them.
+// is set. It reads a page at a time into a memguard.Stream, which keeps what
-func readSecretFromReader(reader io.Reader) ([]bufferInfo, int, error) {
+// it is given encrypted, so that the value is in locked memory only once, in
-	var buffers []bufferInfo
+// the buffer returned, and both size checks come before that buffer is
+// allocated. It takes the value back out with Stream.Next, not Stream.Read:
+// Read puts back the part of a piece that does not fit, and memguard hangs
+// when it runs out of locked memory doing that.
+func readSecretFromReader(
+	reader io.Reader, trimNewline bool,
+) (*memguard.LockedBuffer, error) {
+	stream := memguard.NewStream()
 
-	totalSize := 0
+	chunk := memguard.NewBuffer(os.Getpagesize())
-	currentBufferSize := initialBufferSize
+	defer chunk.Destroy()
-	sameSize := 0
+
+	size := 0
+	endsWithNewline := false
 
 	for {
-		// Create a new buffer
+		n, err := io.ReadFull(reader, chunk.Bytes())
-		buffer := memguard.NewBuffer(currentBufferSize)
+		if n > 0 {
-		n, err := io.ReadFull(reader, buffer.Bytes())
+			size += n
-
+			if size > maxSecretSize {
-		if n == 0 {
+				return nil, errSecretTooLarge
-			// No data read, destroy the unused buffer
-			buffer.Destroy()
-		} else {
-			buffers = append(buffers, bufferInfo{buffer: buffer, used: n})
-			totalSize += n
-
-			if totalSize > maxSecretSize {
-				destroyBuffers(buffers)
-
-				return nil, 0, errSecretTooLarge
 			}
 
-			// If we filled the buffer, consider growing for next iteration
+			endsWithNewline = chunk.Bytes()[n-1] == '\n'
-			if n == currentBufferSize {
+
-				currentBufferSize = updateBufferSize(currentBufferSize, &sameSize)
+			// Write encrypts the bytes and wipes them from chunk.
-			}
+			_, _ = stream.Write(chunk.Bytes()[:n])
 		}
 
-		if err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF) {
+		if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
 			break
-		} else if err != nil {
+		}
-			destroyBuffers(buffers)
 
-			return nil, 0, err
+		if err != nil {
+			return nil, err
 		}
 	}
 
-	return buffers, totalSize, nil
+	// Freed before the value's buffer is allocated, so that the two are
+	// never locked at once.
+	chunk.Destroy()
+
+	if trimNewline && endsWithNewline {
+		size--
 	}
 
-// combineBuffers copies the used portions of buffers into a single
+	err := checkLockedMemoryLimit(size)
-// protected buffer of totalSize bytes
+	if err != nil {
-func combineBuffers(buffers []bufferInfo, totalSize int) *memguard.LockedBuffer {
+		return nil, err
-	valueBuffer := memguard.NewBuffer(totalSize)
-
-	offset := 0
-	for _, b := range buffers {
-		copy(valueBuffer.Bytes()[offset:], b.buffer.Bytes()[:b.used])
-		offset += b.used
 	}
 
-	return valueBuffer
+	value := memguard.NewBuffer(size)
+
+	// copy stops at the value's size, so a trimmed newline is not copied.
+	for offset := 0; offset < size; {
+		decrypted, err := stream.Next()
+		if err != nil {
+			value.Destroy()
+
+			return nil, fmt.Errorf("failed to decrypt the value read: %w", err)
+		}
+
+		offset += copy(value.Bytes()[offset:], decrypted.Bytes())
+		decrypted.Destroy()
+	}
+
+	return value, nil
 }
 
 // AddSecret adds a secret to the current vault
@@ -370,35 +404,21 @@ func (cli *Instance) AddSecret(secretName string, force bool) error {
 
 	secret.Debug("Got current vault", "vault_name", vlt.GetName())
 
-	// Read secret value directly into protected buffers
+	// Read the secret value, less a trailing newline, into locked memory
-	secret.Debug("Reading secret value from stdin into protected buffers")
+	secret.Debug("Reading secret value from stdin into a protected buffer")
 
-	buffers, totalSize, err := readSecretFromReader(cli.cmd.InOrStdin())
+	valueBuffer, err := readSecretFromReader(cli.cmd.InOrStdin(), true)
 	if err != nil {
-		if errors.Is(err, errSecretTooLarge) {
+		if errors.Is(err, errSecretTooLarge) || errors.Is(err, errLockedMemoryLimit) {
 			return err
 		}
 
 		return fmt.Errorf("failed to read secret value: %w", err)
 	}
-	defer destroyBuffers(buffers)
-
-	// Check for trailing newline in the last buffer
-	if len(buffers) > 0 && totalSize > 0 {
-		lastBuffer := &buffers[len(buffers)-1]
-		if lastBuffer.buffer.Bytes()[lastBuffer.used-1] == '\n' {
-			lastBuffer.used--
-			totalSize--
-		}
-	}
-
-	secret.Debug("Read secret value from stdin",
-		"value_length", totalSize, "buffers", len(buffers))
-
-	// Combine all buffers into a single protected buffer
-	valueBuffer := combineBuffers(buffers, totalSize)
 	defer valueBuffer.Destroy()
 
+	secret.Debug("Read secret value from stdin", "value_length", valueBuffer.Size())
+
 	// Locked only now that stdin has been read: in `secret encrypt key |
 	// secret add name`, holding the lock while reading would leave each
 	// command waiting for the other.
@@ -667,7 +687,7 @@ func (cli *Instance) ImportSecret(
 
 	vlt.Mnemonic, vlt.UnlockPassphrase = cli.Mnemonic, cli.UnlockPassphrase
 
-	// Read secret value from the source file into protected buffers
+	// Read secret value from the source file into a protected buffer
 	file, err := cli.fs.Open(sourceFile)
 	if err != nil {
 		return fmt.Errorf("failed to open file %s: %w", sourceFile, err)
@@ -679,14 +699,10 @@ func (cli *Instance) ImportSecret(
 		}
 	}()
 
-	buffers, totalSize, err := readSecretFromReader(file)
+	valueBuffer, err := readSecretFromReader(file, false)
 	if err != nil {
 		return fmt.Errorf("failed to read secret from file %s: %w", sourceFile, err)
 	}
-	defer destroyBuffers(buffers)
-
-	// Combine all buffers into a single protected buffer
-	valueBuffer := combineBuffers(buffers, totalSize)
 	defer valueBuffer.Destroy()
 
 	// Locked only now that the file has been read, as in AddSecret: the

internal/cli/secrets_size_test.go

@@ -3,17 +3,23 @@ package cli
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"fmt"
 	"io"
+	"os"
+	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/secret/internal/secret"
 	"sneak.berlin/go/secret/internal/vault"
 	"sneak.berlin/go/secret/pkg/agehd"
 )
@@ -208,23 +214,25 @@ func TestImportSecretVariousSizes(t *testing.T) {
 	}
 }
 
-// TestAddSecretBufferGrowth tests that our buffer growth strategy works correctly
+// TestAddSecretAroundPageSize tests that secret add, which reads its input
+// one memory page (usually 4 KiB) at a time, stores values just under, at and
+// just over one and two pages unchanged.
 //
 //nolint:paralleltest // in parallel, size tests could exceed the memlock limit
-func TestAddSecretBufferGrowth(t *testing.T) {
+func TestAddSecretAroundPageSize(t *testing.T) {
-	// Test various sizes that should trigger buffer growth
+	// Sizes around one and two pages of 4 KiB, then larger ones up to 1 MiB
 	sizes := []int{
 		1,       // Single byte
 		100,     // Small
-		4095,    // Just under initial 4KB
+		4095,    // Just under 4KB
 		4096,    // Exactly 4KB
 		4097,    // Just over 4KB
-		8191,    // Just under 8KB (first double)
+		8191,    // Just under 8KB
 		8192,    // Exactly 8KB
 		8193,    // Just over 8KB
-		12288,   // 12KB (should trigger second double)
+		12288,   // 12KB
 		16384,   // 16KB
-		32768,   // 32KB (after more doublings)
+		32768,   // 32KB
 		65536,   // 64KB
 		131072,  // 128KB
 		524288,  // 512KB
@@ -315,6 +323,144 @@ func TestAddSecretStreamingBehavior(t *testing.T) {
 		"Retrieved secret should match original")
 }
 
+// newBinaryTestStateDir returns a new state directory holding a vault made
+// from the test mnemonic, for the secret binary to use.
+func newBinaryTestStateDir(t *testing.T) string {
+	t.Helper()
+
+	stateDir := t.TempDir()
+	_, err := vault.CreateVault(afero.NewOsFs(), stateDir, "default",
+		testMnemonicBuffer(t), nil)
+	require.NoError(t, err)
+
+	return stateDir
+}
+
+// runAtLockedMemoryLimit runs the secret binary with args, stdin, the state
+// directory stateDir and the test mnemonic, its locked-memory limit (ulimit
+// -l) lowered to limitKiB, and returns what it printed.
+func runAtLockedMemoryLimit(
+	t *testing.T, stateDir string, limitKiB int, stdin io.Reader,
+	args ...string,
+) (string, error) {
+	t.Helper()
+
+	const commandWait = 10 * time.Second
+
+	secretPath, err := filepath.Abs("../../secret")
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(t.Context(), commandWait)
+	defer cancel()
+
+	// The shell lowers the limit for itself and the binary only.
+	script := fmt.Sprintf(`ulimit -l %d && exec "$0" "$@"`, limitKiB)
+
+	//nolint:gosec // G204: runs the freshly built secret binary
+	cmd := exec.CommandContext(ctx, "sh",
+		append([]string{"-c", script, secretPath}, args...)...)
+	cmd.Env = []string{
+		secret.EnvStateDir + "=" + stateDir,
+		secret.EnvMnemonic + "=" + testMnemonic,
+	}
+	cmd.Stdin = stdin
+
+	output, err := cmd.CombinedOutput()
+
+	return string(output), err
+}
+
+// requireRefusedAtLockedMemoryLimit runs the secret binary as
+// runAtLockedMemoryLimit does, and requires it to exit with status 1 and
+// print wantError, with no panic.
+func requireRefusedAtLockedMemoryLimit(
+	t *testing.T, stateDir string, limitKiB int, wantError string,
+	stdin io.Reader, args ...string,
+) {
+	t.Helper()
+
+	output, err := runAtLockedMemoryLimit(t, stateDir, limitKiB, stdin, args...)
+
+	var exitErr *exec.ExitError
+
+	require.ErrorAs(t, err, &exitErr, "%v: %s", args, output)
+	assert.Equal(t, 1, exitErr.ExitCode(), "%v: %s", args, output)
+	assert.Contains(t, output, wantError, args)
+	assert.NotContains(t, output, "panic", args)
+}
+
+// TestSecretsAtLockedMemoryLimit runs the secret binary with the
+// locked-memory limit (ulimit -l) lowered to 1 MiB, where memguard would crash
+// the process with a panic instead of returning an error. add must store the
+// largest value that fits, which takes holding it in locked memory only once.
+// add, import and generate secret must refuse a value one byte larger with
+// exit status 1 and an error saying why.
+func TestSecretsAtLockedMemoryLimit(t *testing.T) {
+	t.Parallel()
+
+	const limitKiB = 1024
+
+	fits := limitKiB*kibibyte - lockedMemoryReservePages*os.Getpagesize()
+	limitError := fmt.Sprintf("locked-memory limit of %d KiB", limitKiB)
+	stateDir := newBinaryTestStateDir(t)
+
+	tooBigFile := filepath.Join(t.TempDir(), "too-big")
+	require.NoError(t, os.WriteFile(tooBigFile, make([]byte, fits+1), 0o600))
+
+	output, err := runAtLockedMemoryLimit(t, stateDir, limitKiB,
+		bytes.NewReader(make([]byte, fits)), "add", "fits")
+	require.NoError(t, err, "the largest value that fits: %s", output)
+
+	requireRefusedAtLockedMemoryLimit(t, stateDir, limitKiB, limitError,
+		bytes.NewReader(make([]byte, fits+1)), "add", "too-big")
+	requireRefusedAtLockedMemoryLimit(t, stateDir, limitKiB, limitError,
+		nil, "import", "too-big", "--source", tooBigFile)
+	requireRefusedAtLockedMemoryLimit(t, stateDir, limitKiB, limitError,
+		nil, "generate", "secret", "too-big", "--length", strconv.Itoa(fits+1))
+}
+
+// TestSecretsUnderSmallLockedMemoryLimits runs add, import and generate
+// secret with a short value under two small locked-memory limits (ulimit
+// -l). Under 20 KiB, too little for the buffers they need besides the value,
+// each must refuse it with exit status 1 and an error naming the limit, where
+// memguard would panic or hang. Under 64 KiB, the default of older Linux
+// kernels, each must store it.
+func TestSecretsUnderSmallLockedMemoryLimits(t *testing.T) {
+	t.Parallel()
+
+	const (
+		tooSmallLimitKiB   = 20
+		oldDefaultLimitKiB = 64
+		shortValue         = "hunter2"
+	)
+
+	tooSmallLimitError := fmt.Sprintf("locked-memory limit of %d KiB",
+		tooSmallLimitKiB)
+	stateDir := newBinaryTestStateDir(t)
+
+	shortFile := filepath.Join(t.TempDir(), "short")
+	require.NoError(t, os.WriteFile(shortFile, []byte(shortValue), 0o600))
+
+	requireRefusedAtLockedMemoryLimit(t, stateDir, tooSmallLimitKiB,
+		tooSmallLimitError, strings.NewReader(shortValue), "add", "added")
+	requireRefusedAtLockedMemoryLimit(t, stateDir, tooSmallLimitKiB,
+		tooSmallLimitError, nil, "import", "imported", "--source", shortFile)
+	requireRefusedAtLockedMemoryLimit(t, stateDir, tooSmallLimitKiB,
+		tooSmallLimitError, nil, "generate", "secret", "generated")
+
+	output, err := runAtLockedMemoryLimit(t, stateDir, oldDefaultLimitKiB,
+		strings.NewReader(shortValue), "add", "added")
+	require.NoError(t, err, "add: %s", output)
+
+	output, err = runAtLockedMemoryLimit(t, stateDir, oldDefaultLimitKiB,
+		nil, "import", "imported", "--source", shortFile)
+	require.NoError(t, err, "import: %s", output)
+
+	output, err = runAtLockedMemoryLimit(t, stateDir, oldDefaultLimitKiB,
+		nil, "generate", "secret", "generated")
+	require.NoError(t, err, "generate secret: %s", output)
+}
+
 // slowReader simulates a reader that returns data in small chunks
 type slowReader struct {
 	data      []byte

internal/cli/unlockers.go

@@ -314,7 +314,8 @@ func (cli *Instance) UnlockersList(jsonOutput bool) error {
 		return err
 	}
 
-	var unlockers []UnlockerInfo
+	// Not nil, so that a vault with no unlockers prints [] in JSON, not null
+	unlockers := make([]UnlockerInfo, 0, len(unlockerMetadata))
 
 	for _, unlockerID := range slices.Sorted(maps.Keys(unlockerMetadata)) {
 		metadata := unlockerMetadata[unlockerID]

internal/cli/unlockers_list_test.go

@@ -1,13 +1,14 @@
 // Unlocker List Tests
 //
-// Tests for `secret unlocker list` behavior when an unlocker's metadata
+// Tests for `secret unlocker list`:
-// cannot be read or used:
 //
 //   - TestUnlockersListToleratesCorruptMetadata: one unlocker's corrupt
 //     metadata does not stop the others from being listed.
 //   - TestUnlockersListSkipsUnreadableMetadata: an unlocker whose metadata
 //     file cannot be checked for or read is left out, and the other is
 //     still listed.
+//   - TestUnlockersListEmptyJSON: a vault with no unlockers lists them as
+//     an empty JSON list.
 
 //nolint:testpackage // white-box test of unexported internals
 package cli
@@ -321,3 +322,36 @@ func TestUnlockersListSkipsUnreadableMetadata(t *testing.T) {
 		})
 	}
 }
+
+// TestUnlockersListEmptyJSON is a regression test for
+// https://git.eeqj.de/sneak/secret/issues/148: listing the unlockers of a
+// vault that has none, as after its last unlocker is removed, printed
+// "unlockers": null in JSON form instead of an empty list.
+func TestUnlockersListEmptyJSON(t *testing.T) {
+	t.Parallel()
+
+	fs := afero.NewMemMapFs()
+	vaultDir := filepath.Join(listTestStateDir, "vaults.d", listTestVaultName)
+
+	require.NoError(t, afero.WriteFile(
+		fs, filepath.Join(listTestStateDir, "currentvault"),
+		[]byte(listTestVaultName), listTestFilePerm,
+	))
+	require.NoError(t, fs.MkdirAll(
+		filepath.Join(vaultDir, listTestUnlockersDirName), listTestDirPerm,
+	))
+
+	var buf bytes.Buffer
+
+	cmd := &cobra.Command{}
+	cmd.SetOut(&buf)
+
+	instance := &Instance{fs: fs, stateDir: listTestStateDir, cmd: cmd}
+	require.NoError(t, instance.UnlockersList(true))
+
+	var decoded map[string]any
+
+	require.NoError(t, json.Unmarshal(buf.Bytes(), &decoded))
+	assert.Equal(t, []any{}, decoded["unlockers"],
+		"no unlockers must be printed as [], not %s", buf.String())
+}

internal/cli/vault_list_test.go

@@ -0,0 +1,42 @@
+package cli_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/secret/internal/cli"
+	"sneak.berlin/go/secret/internal/secret"
+)
+
+// TestVaultListEmptyJSON is a regression test for
+// https://git.eeqj.de/sneak/secret/issues/148: listing the vaults when
+// vaults.d holds none printed "vaults": null in JSON form instead of an
+// empty list.
+func TestVaultListEmptyJSON(t *testing.T) {
+	t.Parallel()
+
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll(
+		filepath.Join(testStateDir, "vaults.d"), secret.DirPerms,
+	))
+
+	var buf bytes.Buffer
+
+	cmd := &cobra.Command{}
+	cmd.SetOut(&buf)
+
+	c := cli.NewCLIInstanceWithStateDir(fs, testStateDir)
+	require.NoError(t, c.ListVaults(cmd, true))
+
+	var decoded map[string]any
+
+	require.NoError(t, json.Unmarshal(buf.Bytes(), &decoded))
+	assert.Equal(t, []any{}, decoded["vaults"],
+		"no vaults must be printed as [], not %s", buf.String())
+}

internal/cli/version.go

@@ -229,9 +229,13 @@ func printVersionRow(
 	}
 
 	// Determine status
-	status := "expired"
+	status := "inactive"
-	if version == currentVersion {
+
+	switch {
+	case version == currentVersion:
 		status = "current"
+	case sv.Metadata.NotAfter != nil && sv.Metadata.NotAfter.Before(time.Now()):
+		status = "expired"
 	}
 
 	// Format timestamps

internal/cli/version_test.go

@@ -5,6 +5,8 @@
 // - TestListVersionsCommand: Tests `secret version list` command output
 // - TestListVersionsNonExistentSecret: Tests error handling for missing secrets
 // - TestPromoteVersionCommand: Tests `secret version promote` command
+// - TestListVersionsStatusAfterPromote: Tests each version's status after
+//   promoting an older version
 // - TestPromoteNonExistentVersion: Tests error handling for invalid promotion
 // - TestPromoteHelpSaysCurrentFileIsRewritten: Tests `secret version promote
 //   --help` text
@@ -244,6 +246,66 @@ func TestPromoteVersionCommand(t *testing.T) {
 	assert.Equal(t, []byte("version-1"), promoted.Bytes())
 }
 
+func TestListVersionsStatusAfterPromote(t *testing.T) {
+	t.Parallel()
+
+	fs := afero.NewMemMapFs()
+	cli := NewCLIInstanceWithStateDir(fs, testStateDir)
+	cli.Mnemonic = testMnemonicBuffer(t)
+
+	setupTestVault(t, fs)
+
+	vlt, err := vault.GetCurrentVault(fs, testStateDir)
+	require.NoError(t, err)
+
+	vlt.Mnemonic = cli.Mnemonic
+
+	addTestSecret(t, vlt, []byte("version-1"), false)
+	addTestSecret(t, vlt, []byte("version-2"), true)
+	addTestSecret(t, vlt, []byte("version-3"), true)
+
+	vaultDir, err := vlt.GetDirectory()
+	require.NoError(t, err)
+
+	// Newest first
+	versions, err := secret.ListVersions(fs,
+		filepath.Join(vaultDir, "secrets.d", "test%secret"))
+	require.NoError(t, err)
+	require.Len(t, versions, 3)
+
+	newest, middle, oldest := versions[0], versions[1], versions[2]
+
+	cmd := newRootCmd()
+
+	var buf bytes.Buffer
+	cmd.SetOut(&buf)
+	cmd.SetErr(&buf)
+
+	err = cli.PromoteVersion(cmd, "test/secret", oldest)
+	require.NoError(t, err)
+
+	buf.Reset()
+
+	err = cli.ListVersions(cmd, "test/secret")
+	require.NoError(t, err)
+
+	// STATUS is the fourth field of a row, since CREATED is a date and a time
+	statuses := map[string]string{}
+
+	for line := range strings.SplitSeq(buf.String(), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) > 3 {
+			statuses[fields[0]] = fields[3]
+		}
+	}
+
+	assert.Equal(t, "current", statuses[oldest])
+	// Its NOT_AFTER was set when the newest version was added
+	assert.Equal(t, "expired", statuses[middle])
+	// It has no NOT_AFTER
+	assert.Equal(t, "inactive", statuses[newest])
+}
+
 func TestPromoteNonExistentVersion(t *testing.T) {
 	t.Parallel()
 

internal/secret/version.go

@@ -32,7 +32,8 @@ type VersionMetadata struct {
 	ID        string     `json:"id"`                  // ULID
 	CreatedAt *time.Time `json:"createdAt,omitempty"` // When version was created
 	NotBefore *time.Time `json:"notBefore,omitempty"` // When this version becomes active
-	NotAfter  *time.Time `json:"notAfter,omitempty"`  // Expiry (nil = current)
+	// NotAfter is set when another version is added while this one is current
+	NotAfter *time.Time `json:"notAfter,omitempty"`
 }
 
 // Version represents a version of a secret

internal/vault/management.go

@@ -140,8 +140,9 @@ func ListVaults(fs afero.Fs, stateDir string) ([]string, error) {
 		return nil, fmt.Errorf("failed to read vaults directory: %w", err)
 	}
 
-	// Extract vault names
+	// Extract vault names. Not nil, so that no vaults prints [] in JSON,
-	var vaults []string
+	// not null.
+	vaults := make([]string, 0, len(entries))
 
 	for _, entry := range entries {
 		if entry.IsDir() {