Keep Go's build cache between builds (closes #124)

The Dockerfile runs make test and make build with Go's build cache in a
BuildKit cache mount, so a build compiles only what changed since the
last one instead of the standard library and every dependency from
nothing. The mount has an id of its own, so builds of other repositories,
compiled against other C headers, do not share it. script/test passes
-count=1, so no test result is taken from the cache.

Model: opus-5-5

Dockerfile

@@ -50,15 +50,26 @@ ARG CHECK_EPOCH
 
 COPY . .
 
-RUN make test
+# Go's build cache is kept between builds in this cache mount, which make test
+# and make build both use, so each compiles only what changed since the last
+# build, not the standard library and every dependency from nothing.
+# script/test passes -count=1, so test results are never taken from it.
+# The mount has its own id because the default id, its path, is shared with
+# other repositories' builds, and Go's cache does not notice changes to C
+# libraries: an entry compiled there against other C headers could be reused
+# here. It does not notice a change of this image's own C headers either.
+RUN --mount=type=cache,id=sneak/secret/go-build,target=/root/.cache/go-build \
+    make test
 
 # The version stamped into the binary: the VERSION build argument when one
 # is given, otherwise `git describe --tags --always` of the .git the build
 # context carries: the tag on a tagged commit, tag-N-gHASH on a commit after
 # one, the short commit when no tag is reachable. A context that carries .git
-# and still yields no version fails the build.
+# and still yields no version fails the build. make build uses the same Go
+# build cache mount as make test.
 ARG VERSION
-RUN version="${VERSION:-$(git describe --tags --always)}"; \
+RUN --mount=type=cache,id=sneak/secret/go-build,target=/root/.cache/go-build \
+    version="${VERSION:-$(git describe --tags --always)}"; \
     if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
         [ "$version" = unknown ]; }; then \
         echo "no version could be derived although the build context carries .git" >&2; \

README.md

@@ -604,7 +604,8 @@ provide:
 - `script/build` — build the `secret` binary into the repo root, stamping the
   version (`VERSION` from the environment, else `git describe`) and the git
   commit
-- `script/test` — run `go vet` and the test suite (verbose rerun on failure)
+- `script/test` — run `go vet` and the test suite (verbose rerun on failure),
+  every test on every run, never a result from Go's test cache
 - `script/lint` — run `golangci-lint` in docker only: builds `Dockerfile.lint`,
   where the linter is a build step that runs on every call, also on an unchanged
   tree
@@ -624,7 +625,8 @@ provide:
 - `script/cibuild` — CI entrypoint: `docker build --ulimit memlock=-1:-1 .`
   (memguard needs mlock; the Dockerfile runs the checks), with a new
   `CHECK_EPOCH` build argument on every run so the checks run again on an
-  unchanged tree
+  unchanged tree; the `Dockerfile` keeps Go's build cache between builds, so
+  `make test` and `make build` compile only what changed
 - `script/precommit` — pre-commit checks: `go mod tidy` verification, then
   `script/check`
 - `script/install-precommit` — install the git pre-commit hook that runs

TODO.md

@@ -18,6 +18,32 @@ https://git.eeqj.de/sneak/secret/milestone/12
 
 # Completed Steps
 
+- 2026-10-06: `make test` in `script/cibuild` no longer compiles the standard
+  library and every dependency from nothing on every build
+  (https://git.eeqj.de/sneak/secret/issues/124). The `Dockerfile` runs it and
+  `make build` with Go's build cache in a BuildKit cache mount, which docker
+  keeps between builds, locally and on the Gitea runner alike, so each compiles
+  only what changed since the last build. The mount has an id of its own, so
+  other repositories' builds do not share it. `script/test` passes `-count=1`,
+  so every test runs on every build and no result comes from Go's test cache. A
+  build with an empty cache, such as the first after docker's build cache is
+  cleared, compiles everything in `make test` as before.
+- 2026-10-06: The tests run quickly with the race detector on
+  (https://git.eeqj.de/sneak/secret/issues/120). Most of their time went to
+  deriving keys from passphrases with scrypt, which is slow on purpose. The new
+  `secret.ScryptWorkFactor`, when not zero, replaces age's scrypt work factor
+  when a passphrase encrypts; the tests of `internal/secret`, `internal/vault`
+  and `internal/cli` set it to 1 before any test runs, and the program never
+  sets it; `TestGetCommandOutputsToStdout` checks that the passphrase unlocker
+  the built binary's `secret init` writes names age's work factor, 18.
+  `TestRemovalAsksWithoutHoldingLock` and `TestFailedCommandReleasesLock` no
+  longer run in parallel with other tests: each waits at most 10 seconds for the
+  in-memory lock that every test in the package shares, and other tests'
+  commands held it longer. `TestConcurrentAddsKeepEveryVersion`, which times
+  nothing, and `TestGetCommandOutputsToStdout`, which no longer sets an
+  environment variable its commands do not read, now run in parallel. The
+  `script/cibuild` comment no longer says that tests are skipped without its
+  memlock ulimit.
 - 2026-10-05: No test stores a secret larger than 1 MiB
   (https://git.eeqj.de/sneak/secret/issues/52). The size tests for `secret add`,
   `secret import` and the stdin buffer no longer try 2 MB, 10 MB, 99 MB, 100 MB

internal/cli/confirm_test.go

@@ -353,9 +353,9 @@ func TestRemovalWithoutTerminalFailsAtOnce(t *testing.T) {
 // for its answer, another command can take the state directory lock and
 // change the secret, and that the removal then removes nothing, since the
 // secret is no longer what the question named.
+//
+//nolint:paralleltest // times commands against the in-memory lock all tests share
 func TestRemovalAsksWithoutHoldingLock(t *testing.T) {
-	t.Parallel()
-
 	r := newRemoval(t, "rm")
 
 	answers, answerWriter := io.Pipe()

internal/cli/integration_test.go

@@ -52,8 +52,12 @@ func runSecretWithStdin(stdin string, env map[string]string, args ...string) (st
 	return cli.ExecuteCommandInProcess(args, stdin, env)
 }
 
-// TestMain runs before all tests and ensures the binary is built
+// TestMain runs before all tests and ensures the binary is built. It also
+// makes passphrase encryption in the tests cheap (see
+// secret.ScryptWorkFactor); the binary keeps age's work factor.
 func TestMain(m *testing.M) {
+	secret.ScryptWorkFactor = 1
+
 	// Get the current working directory
 	wd, err := os.Getwd()
 	if err != nil {

internal/cli/lock_test.go

@@ -94,9 +94,9 @@ func numbered(prefix string, count int) []string {
 // lock, adds of a new secret all find it absent and replace each other, and
 // forced adds read the same highest version number and overwrite each
 // other's version. With it they behave as if run one after another.
-//
-//nolint:paralleltest // times commands against the in-memory lock all tests share
 func TestConcurrentAddsKeepEveryVersion(t *testing.T) {
+	t.Parallel()
+
 	mnemonic := testMnemonicBuffer(t)
 
 	const adds = 8
@@ -110,6 +110,8 @@ func TestConcurrentAddsKeepEveryVersion(t *testing.T) {
 		{"real", afero.NewOsFs(), t.TempDir()},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
 			_, err := vault.CreateVault(tc.fs, tc.stateDir, "default", mnemonic, nil)
 			require.NoError(t, err)
 
@@ -235,9 +237,9 @@ func TestEncryptPipedIntoAdd(t *testing.T) {
 
 // TestFailedCommandReleasesLock checks that a command failing after it
 // took the state directory lock leaves the lock free for the next command.
+//
+//nolint:paralleltest // times commands against the in-memory lock all tests share
 func TestFailedCommandReleasesLock(t *testing.T) {
-	t.Parallel()
-
 	fs := afero.NewMemMapFs()
 	cli := NewCLIInstanceWithStateDir(fs, testStateDir)
 

internal/cli/stdout_stderr_test.go

@@ -15,11 +15,11 @@ import (
 // TestGetCommandOutputsToStdout tests that 'secret get' outputs the secret
 // value to stdout, not stderr
 func TestGetCommandOutputsToStdout(t *testing.T) {
-	// Create a temporary directory for our vault
-	tempDir := t.TempDir()
+	t.Parallel()
 
-	// Set environment variables for the test
-	t.Setenv(secret.EnvStateDir, tempDir)
+	// Create a temporary directory for our vault; each command is given it
+	// in its environment
+	tempDir := t.TempDir()
 
 	// Find the secret binary path
 	wd, err := filepath.Abs("../..")
@@ -41,6 +41,18 @@ func TestGetCommandOutputsToStdout(t *testing.T) {
 	output, err := cmd.CombinedOutput()
 	require.NoError(t, err, "init should succeed: %s", string(output))
 
+	// The binary, unlike these tests, encrypts the passphrase unlocker's key
+	// at age's scrypt work factor, 18. age writes the work factor last on the
+	// second line of priv.age: "-> scrypt <salt> <work factor>".
+	vaultDir := filepath.Join(tempDir, "vaults.d", "default")
+	unlockerName := readFile(t, filepath.Join(vaultDir, "current-unlocker"))
+	unlockerDir := filepath.Join(vaultDir, "unlockers.d", string(unlockerName))
+	privAge := readFile(t, filepath.Join(unlockerDir, "priv.age"))
+	header := strings.SplitN(string(privAge), "\n", 3)
+	require.Len(t, header, 3, "priv.age should start with an age header")
+	assert.Regexp(t, `^-> scrypt \S+ 18$`, header[1],
+		"the passphrase unlocker should be encrypted at scrypt work factor 18")
+
 	// Add a secret
 	//nolint:gosec // G204: test executes the freshly built secret binary
 	cmd = exec.CommandContext(t.Context(), secretPath, "add", "test/secret")

internal/secret/crypto.go

@@ -28,6 +28,17 @@ var (
 // terminal to read the mnemonic from, reading it failed, or it was empty.
 var ErrMnemonicNotRead = errors.New("failed to read mnemonic")
 
+// ScryptWorkFactor is, when not zero, the scrypt work factor that
+// EncryptWithPassphrase uses instead of age's, 18: log2 of scrypt's cost
+// parameter N. Deriving a key with age's takes about a second and 256 MiB, on
+// purpose, since so does every guess at the passphrase. Only tests set it,
+// lower, before any test runs, so that the passphrase unlockers they create
+// cost nothing; the program leaves it zero. Decryption takes the work factor
+// from the encrypted data, so it needs no setting.
+//
+//nolint:gochecknoglobals // set by the tests of the packages that use this one
+var ScryptWorkFactor int
+
 // EncryptToRecipient encrypts data to a recipient using age
 // The data parameter should be a LockedBuffer for secure memory handling
 func EncryptToRecipient(
@@ -142,6 +153,10 @@ func EncryptWithPassphrase(
 		return nil, fmt.Errorf("failed to create scrypt recipient: %w", err)
 	}
 
+	if ScryptWorkFactor != 0 {
+		recipient.SetWorkFactor(ScryptWorkFactor)
+	}
+
 	return EncryptToRecipient(data, recipient)
 }
 

internal/secret/secret_test.go

@@ -25,6 +25,14 @@ var (
 	errNotImplementedInMock = errors.New("not implemented in mock")
 )
 
+// TestMain makes passphrase encryption in the tests cheap; see
+// ScryptWorkFactor.
+func TestMain(m *testing.M) {
+	ScryptWorkFactor = 1
+
+	os.Exit(m.Run())
+}
+
 // MockVault is a test implementation of the VaultInterface
 type MockVault struct {
 	name            string

internal/vault/vault_test.go

@@ -3,6 +3,7 @@ package vault_test
 import (
 	"bytes"
 	"errors"
+	"os"
 	"path/filepath"
 	"slices"
 	"testing"
@@ -28,6 +29,14 @@ const (
 	testPassphrase = "test-passphrase"
 )
 
+// TestMain makes passphrase encryption in the tests cheap; see
+// secret.ScryptWorkFactor.
+func TestMain(m *testing.M) {
+	secret.ScryptWorkFactor = 1
+
+	os.Exit(m.Run())
+}
+
 // testMnemonicBuffer returns testMnemonic in a locked buffer that is
 // destroyed when the test ends.
 func testMnemonicBuffer(t *testing.T) *memguard.LockedBuffer {

script/cibuild

@@ -1,12 +1,13 @@
 #!/bin/sh
 # script/cibuild: run the CI build. The Dockerfile runs script/check
 # (via make check), so a successful build implies all checks pass.
-# The Gitea workflow runs this on push. The memlock ulimit lets the tests
-# that lock large secrets in memory (memguard mlocks them) run; under the
-# lower limit of a plain `docker build .` they are skipped.
+# The Gitea workflow runs this on push. The memlock ulimit lifts the limit
+# on memory the tests lock (memguard mlocks secrets); they also pass under
+# the lower limit of a plain `docker build .`.
 # A cached build checks nothing: a new CHECK_EPOCH on every run makes the
 # Dockerfile's check steps run again on an unchanged tree, while its base
-# images and module downloads stay cached.
+# images, module downloads and the Go build cache that make test and make
+# build use stay cached.
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"

script/test

@@ -9,7 +9,9 @@ main() {
     # CGO is required (Makefile exports this too)
     export CGO_ENABLED=1
     go vet ./...
-    go test ./... || go test -v ./...
+    # -count=1: run every test, never take a result from Go's test cache,
+    # which the Dockerfile keeps between builds
+    go test -count=1 ./... || go test -count=1 -v ./...
 }
 
 main "$@"