Keep make test under 2 GB of memory (closes #344 )

Every test that starts a database seeds the admin account, hashing its password with Argon2id at 64 MB, about 150 MB under -race, and internal/handlers and internal/database ran dozens of those at once. That was the memory, and most of internal/handlers' run time; the product code does not leak. HashPassword now hashes at a 1 MB cost only when testing.Testing() reports a test binary, so every test package gets it with nothing to add and a binary built by go build always hashes at the shipped parameters. TestHashPassword_ShippedParameters still hashes and verifies through HashPassword at the shipped cost. script/test adds -p 4 -parallel 8. Model: opus-5-5
2026-10-02 03:02:28 +02:00
11 changed files with 114 additions and 271 deletions
@@ -5,6 +5,7 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	"testing"
 	"time"

 	"go.uber.org/fx"
@@ -79,3 +80,14 @@ func (d *Database) ExportSetBannerOut(w io.Writer) {
 func DummyPasswordHashForTest() string {
 	return dummyPasswordHash()
 }
+
+// HashAtShippedCostForTest makes HashPassword hash at the shipped
+// memory cost until t ends. t must not run in parallel with other
+// tests, which would hash at that cost alongside it.
+func HashAtShippedCostForTest(t *testing.T) {
+	t.Helper()
+
+	hashAtShippedCostInTest = true
+
+	t.Cleanup(func() { hashAtShippedCostInTest = false })
+}
@@ -9,6 +9,7 @@ import (
 	"math/big"
 	"strings"
 	"sync"
+	"testing"

 	"golang.org/x/crypto/argon2"
 )
@@ -63,10 +64,30 @@ func DefaultPasswordConfig() *PasswordConfig {
 	}
 }

-// HashPassword generates an Argon2id hash of the password
+// testArgon2Memory is the Argon2id memory cost, in KiB, that a test
+// binary hashes with: 1 MB instead of the shipped 64 MB. Every test
+// that starts a database hashes the bootstrap admin password, dozens
+// of them run in parallel, and under the race detector each 64 MB hash
+// holds about 150 MB. VerifyPassword reads the cost from the hash it
+// checks, so verification follows.
+const testArgon2Memory = 1024
+
+// hashAtShippedCostInTest makes a test binary hash at the shipped
+// memory cost. Only TestHashPassword_ShippedParameters sets it.
+//
+//nolint:gochecknoglobals // set by one test, see above
+var hashAtShippedCostInTest bool
+
+// HashPassword generates an Argon2id hash of the password. A binary
+// built by go test hashes at testArgon2Memory; one built by go build
+// always hashes at the defaults.
 func HashPassword(password string) (string, error) {
 	config := DefaultPasswordConfig()

+	if testing.Testing() && !hashAtShippedCostInTest {
+		config.Memory = testArgon2Memory
+	}
+
 	// Generate a salt
 	salt := make([]byte, config.SaltLen)

@@ -192,6 +192,39 @@ func TestHashPasswordUniqueness(t *testing.T) {
 	}
 }

+// TestHashPassword_ShippedParameters hashes and verifies through
+// HashPassword at the shipped Argon2id parameters. Every other test
+// hashes at the lower memory cost a test binary uses, so this is the
+// one that keeps production hashing covered. One hash and one
+// verification: each costs 64 MB.
+//
+//nolint:paralleltest // changes the hashing cost for the whole binary
+func TestHashPassword_ShippedParameters(t *testing.T) {
+	database.HashAtShippedCostForTest(t)
+
+	password := "correct horse battery staple"
+
+	hash, err := database.HashPassword(password)
+	if err != nil {
+		t.Fatalf("hashing with the shipped parameters: %v", err)
+	}
+
+	const shipped = "$argon2id$v=19$m=65536,t=1,p=4$"
+
+	if !strings.HasPrefix(hash, shipped) {
+		t.Errorf("hash = %q, want prefix %q", hash, shipped)
+	}
+
+	valid, err := database.VerifyPassword(password, hash)
+	if err != nil {
+		t.Fatalf("VerifyPassword() error = %v", err)
+	}
+
+	if !valid {
+		t.Error("VerifyPassword() returned false for correct password")
+	}
+}
+
 // TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
 // path. Login charges an unknown username a verification against a
 // dummy hash so that a nonexistent account is not answered in
@@ -40,6 +40,11 @@ const (
 	ExportPendingSweepMinAge = pendingSweepMinAge
 )

+// ExportIsBlockedIP exposes isBlockedIP for testing.
+func ExportIsBlockedIP(ip net.IP) bool {
+	return isBlockedIP(ip)
+}
+
 // NewTestGuard builds an SSRF Guard from an explicit egress
 // allowlist, without going through config. Passing no prefixes
 // yields the default guard, which blocks every private/reserved
@@ -65,11 +70,6 @@ func ExportBlockedNetworks() []*net.IPNet {
 	return blockedNetworks
 }

-// ExportBlockedPublicNetworks exposes blockedPublicNetworks.
-func ExportBlockedPublicNetworks() []*net.IPNet {
-	return blockedPublicNetworks
-}
-
 // ExportIsForwardableHeader exposes isForwardableHeader.
 func ExportIsForwardableHeader(name string) bool {
 	return isForwardableHeader(name)
@@ -25,16 +25,8 @@ var (
 	errNoIPs      = errors.New(
 		"hostname resolved to no IP addresses",
 	)
-	// ErrBlockedPrivateOrReservedIP reports an address in the
-	// default blocklist's private and reserved ranges,
-	// blockedNetworks.
-	ErrBlockedPrivateOrReservedIP = errors.New(
-		"blocked private or reserved address",
-	)
-	// errBlockedPublicMetadata reports a public address on the
-	// default blocklist, one in blockedPublicNetworks.
-	errBlockedPublicMetadata = errors.New(
-		"blocked cloud metadata address",
+	errBlockedIP = errors.New(
+		"blocked private, reserved or cloud metadata address",
 	)
 	errBlockedMetadata = errors.New(
 		"blocked link-local or cloud instance metadata " +
@@ -45,31 +37,21 @@ var (
 	)
 )

-// blockedNetworks and blockedPublicNetworks together are the
-// default blocklist: the private and reserved IP ranges, plus
-// the public cloud metadata addresses, that are blocked to
-// prevent SSRF attacks. An operator can permit specific blocks
-// out of this set with ALLOWED_EGRESS_CIDRS; see Guard.
-//
-// blockedNetworks holds the private and reserved IP ranges.
-//
-//nolint:gochecknoglobals // package-level network list is appropriate here
-var blockedNetworks []*net.IPNet
-
-// blockedPublicNetworks holds the default blocklist's public
-// addresses, kept apart from blockedNetworks so that they are
-// refused as cloud metadata addresses, never as private or
-// reserved ones.
+// blockedNetworks is the default blocklist: the private and
+// reserved IP ranges, plus the public cloud metadata addresses,
+// that are blocked to prevent SSRF attacks. An operator can
+// permit specific blocks out of this set with
+// ALLOWED_EGRESS_CIDRS; see Guard.
 //
 // A public address belongs on the default blocklist only if it
 // hands credentials, user data or bootstrap material to whatever
-// can reach it, without the caller presenting anything; it goes
-// in this list. A provider's other public addresses are not
-// refused, since reaching them can be legitimate and no list of
-// them could be complete.
+// can reach it, without the caller presenting anything. A
+// provider's other public addresses are not refused, since
+// reaching them can be legitimate and no list of them could be
+// complete.
 //
 //nolint:gochecknoglobals // package-level network list is appropriate here
-var blockedPublicNetworks []*net.IPNet
+var blockedNetworks []*net.IPNet

 // alwaysBlockedNetworks are the ranges no configuration can
 // open: the link-local blocks and the cloud instance metadata
@@ -106,8 +88,8 @@ var blockedPublicNetworks []*net.IPNet
 // when it clears both halves. Nothing in this list can be
 // reopened, so putting a public address here leaves the operator
 // no escape hatch at all — the condition ALLOWED_EGRESS_CIDRS
-// exists to remove. Default-block it in blockedPublicNetworks
-// instead, which an allowlist can override.
+// exists to remove. Default-block it in blockedNetworks instead,
+// which an allowlist can override.
 //
 // This is a criterion, not an enumeration of every metadata
 // address in existence.
@@ -148,9 +130,6 @@ func init() {
 		"::1/128",
 		"fc00::/7",
 		"fe80::/10",
-	})
-
-	blockedPublicNetworks = mustParseCIDRs([]string{
 		// Azure WireServer, a public address that serves VM credentials.
 		"168.63.129.16/32",
 	})
@@ -246,6 +225,13 @@ func matchesAny(networks []*net.IPNet, ip net.IP) bool {
 	return false
 }

+// isBlockedIP checks whether an IP address falls within
+// the default blocklist, before any operator allowlist is
+// considered.
+func isBlockedIP(ip net.IP) bool {
+	return matchesAny(blockedNetworks, ip)
+}
+
 // Guard makes every SSRF decision in the process.
 //
 // It holds the operator's ALLOWED_EGRESS_CIDRS allowlist and
@@ -346,8 +332,7 @@ func (g *Guard) allows(ip net.IP) bool {
 //     consulted, so no configured CIDR reaches link-local or a
 //     cloud metadata endpoint at a non-public address.
 //  2. The allowlist is consulted next, so a listed private
-//     network, or a listed public address on the default
-//     blocklist, becomes reachable.
+//     network becomes reachable.
 //  3. Everything else keeps the default blocklist's answer.
 func (g *Guard) checkIP(ip net.IP) error {
 	if matchesAny(alwaysBlockedNetworks, ip) {
@@ -360,15 +345,9 @@ func (g *Guard) checkIP(ip net.IP) error {
 		return nil
 	}

-	if matchesAny(blockedNetworks, ip) {
+	if isBlockedIP(ip) {
 		return fmt.Errorf(
-			"target IP %s: %w", ip, ErrBlockedPrivateOrReservedIP,
-		)
-	}
-
-	if matchesAny(blockedPublicNetworks, ip) {
-		return fmt.Errorf(
-			"target IP %s: %w", ip, errBlockedPublicMetadata,
+			"target IP %s: %w", ip, errBlockedIP,
 		)
 	}

@@ -7,7 +7,6 @@ import (
 	"net/http/httptest"
 	"net/netip"
 	"net/url"
-	"slices"
 	"testing"
 	"time"

@@ -24,10 +23,6 @@ const (
 	metadataIP  = "169.254.169.254"
 	metadataURL = "http://" + metadataIP + "/latest/meta-data/"

-	// linkLocalIPv4 is the IPv4 link-local block, which holds
-	// metadataIP.
-	linkLocalIPv4 = "169.254.0.0/16"
-
 	// loopbackHookURL is a target on this host: blocked by
 	// default, reachable only once an operator allowlists
 	// loopback.
@@ -242,7 +237,7 @@ func linkLocalRefusedCases() []metadataAlwaysRefusedCase {
 		},
 		{
 			name:   "whole link-local block",
-			allow:  linkLocalIPv4,
+			allow:  "169.254.0.0/16",
 			target: metadataURL,
 		},
 		{
@@ -417,9 +412,6 @@ func TestGuardAllowlist_AzureWireServerReopenable(t *testing.T) {
 		"WireServer must be refused by the default blocklist, "+
 			"which an allowlist can override",
 	)
-	require.NotErrorIs(t, err, delivery.ErrBlockedPrivateOrReservedIP,
-		"WireServer is public, not private or reserved",
-	)

 	assertDialRefused(t, defaultGuard, target)

@@ -504,7 +496,7 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
 	want := []string{
 		// IPv4 link-local: the 169.254.169.254 metadata
 		// service on AWS, Azure and others.
-		linkLocalIPv4,
+		"169.254.0.0/16",
 		// IPv6 link-local.
 		"fe80::/10",
 		// AWS IPv6 IMDS, inside the ULA space an operator may
@@ -534,76 +526,6 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
 	assert.Equal(t, want, got)
 }

-// TestDefaultBlocklist_PinnedSet pins the default blocklist, its
-// private and reserved ranges and its public addresses together,
-// and how ALLOWED_EGRESS_CIDRS opens each entry: listing an entry
-// opens it unless the unconditional set also holds it.
-func TestDefaultBlocklist_PinnedSet(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		cidr       string
-		reopenable bool
-	}{
-		{"127.0.0.0/8", true},
-		{"10.0.0.0/8", true},
-		{"172.16.0.0/12", true},
-		{"192.168.0.0/16", true},
-		{linkLocalIPv4, false},
-		{"0.0.0.0/8", true},
-		{"100.64.0.0/10", true},
-		{"192.0.0.0/24", true},
-		{"192.0.2.0/24", true},
-		{"198.18.0.0/15", true},
-		{"198.51.100.0/24", true},
-		{"203.0.113.0/24", true},
-		{"224.0.0.0/4", true},
-		{"240.0.0.0/4", true},
-		{"::1/128", true},
-		{"fc00::/7", true},
-		{"fe80::/10", false},
-		{"168.63.129.16/32", true},
-	}
-
-	want := make([]string, 0, len(tests))
-	for _, tt := range tests {
-		want = append(want, tt.cidr)
-	}
-
-	nets := slices.Concat(
-		delivery.ExportBlockedNetworks(),
-		delivery.ExportBlockedPublicNetworks(),
-	)
-
-	got := make([]string, 0, len(nets))
-	for _, n := range nets {
-		got = append(got, n.String())
-	}
-
-	assert.ElementsMatch(t, want, got)
-
-	for _, tt := range tests {
-		t.Run(tt.cidr, func(t *testing.T) {
-			t.Parallel()
-
-			prefix := netip.MustParsePrefix(tt.cidr)
-			ip := net.IP(prefix.Addr().AsSlice())
-
-			require.Error(t,
-				delivery.NewTestGuard().ExportCheckIP(ip),
-				"the default guard must refuse %s", ip,
-			)
-
-			err := delivery.NewTestGuard(prefix).ExportCheckIP(ip)
-			if tt.reopenable {
-				assert.NoError(t, err, "listing %s must open it", tt.cidr)
-			} else {
-				assert.Error(t, err, "listing %s must not open it", tt.cidr)
-			}
-		})
-	}
-}
-
 // requireLoopback fails the test unless rawURL's host is a
 // loopback address, so the allowlist test cannot silently stop
 // exercising a blocked range.
@@ -10,7 +10,7 @@ import (
 	"sneak.berlin/go/webhooker/internal/delivery"
 )

-func TestGuardCheckIP_PrivateRanges(t *testing.T) {
+func TestIsBlockedIP_PrivateRanges(t *testing.T) {
 	t.Parallel()

 	tests := []struct {
@@ -56,14 +56,12 @@ func TestGuardCheckIP_PrivateRanges(t *testing.T) {
 				"failed to parse IP %s", tt.ip,
 			)

-			refused := delivery.NewTestGuard().ExportCheckIP(ip) != nil
-
 			assert.Equal(t,
 				tt.blocked,
-				refused,
-				"default guard refuses %s = %v, want %v",
+				delivery.ExportIsBlockedIP(ip),
+				"isBlockedIP(%s) = %v, want %v",
 				tt.ip,
-				refused,
+				delivery.ExportIsBlockedIP(ip),
 				tt.blocked,
 			)
 		})
@@ -1577,22 +1577,11 @@ func (h *Handlers) validateTargetURL(
 			"url", delivery.MaskURL(targetURL),
 			"error", err,
 		)
-
-		msg := "Invalid target URL: " + err.Error()
-
-		// Only a private or reserved address's refusal says how
-		// to allow it. Metadata refusals never do: link-local and
-		// the other unconditional metadata addresses cannot be
-		// opened, and the default blocklist's public addresses,
-		// which listing does open, hand out credentials.
-		if errors.Is(err, delivery.ErrBlockedPrivateOrReservedIP) {
-			msg += ". Private and reserved addresses are refused " +
-				"by default; the server's ALLOWED_EGRESS_CIDRS " +
-				"setting allows named networks (see \"Allowing " +
-				"egress to your own network\" in the README)."
-		}
-
-		http.Error(w, msg, http.StatusBadRequest)
+		http.Error(
+			w,
+			"Invalid target URL: "+err.Error(),
+			http.StatusBadRequest,
+		)

 		return err
 	}
@@ -1,116 +0,0 @@
-package handlers_test
-
-import (
-	"net/http"
-	"net/url"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/webhooker/internal/database"
-)
-
-// privateRefusalHint is the sentence that tells an operator a private
-// destination is refused on purpose, and how to allow one.
-const privateRefusalHint = "Private and reserved addresses are " +
-	"refused by default; the server's ALLOWED_EGRESS_CIDRS setting " +
-	"allows named networks (see \"Allowing egress to your own " +
-	"network\" in the README)."
-
-// TestTargetRefusal_PrivateDestinationSaysHowToAllowIt covers both
-// target types that take a URL, on add and on edit.
-func TestTargetRefusal_PrivateDestinationSaysHowToAllowIt(
-	t *testing.T,
-) {
-	t.Parallel()
-
-	env := setupSourceTest(t)
-
-	targetTypes := []database.TargetType{
-		database.TargetTypeHTTP,
-		database.TargetTypeSlack,
-	}
-
-	for _, targetType := range targetTypes {
-		t.Run(string(targetType), func(t *testing.T) {
-			t.Parallel()
-
-			webhook := seedWebhookWithRetention(t, env.db, 30)
-			targetsPath := "/source/" + webhook.ID + "/targets"
-
-			form := url.Values{}
-			form.Set("name", "private")
-			form.Set("type", string(targetType))
-			form.Set("url", editBlockedURL)
-
-			added := serveTarget(
-				env, http.MethodPost, targetsPath, form,
-			)
-			assert.Equal(t, http.StatusBadRequest, added.Code)
-			assert.Contains(
-				t, added.Body.String(), privateRefusalHint,
-			)
-
-			form.Set("url", editOriginalURL)
-
-			created := serveTarget(
-				env, http.MethodPost, targetsPath, form,
-			)
-			require.Equal(
-				t, http.StatusSeeOther, created.Code,
-				created.Body.String(),
-			)
-
-			targets := targetsForWebhook(t, env.db, webhook.ID)
-			require.Len(t, targets, 1)
-
-			form.Set("url", editBlockedURL)
-
-			edited := submitTargetEdit(
-				env, webhook.ID, targets[0].ID, form,
-			)
-			assert.Equal(t, http.StatusBadRequest, edited.Code)
-			assert.Contains(
-				t, edited.Body.String(), privateRefusalHint,
-			)
-		})
-	}
-}
-
-// TestTargetRefusal_MetadataDestinationDoesNotSayHowToAllowIt: no
-// setting opens a link-local address, and Azure's WireServer hands out
-// VM credentials, so neither refusal points at the setting.
-func TestTargetRefusal_MetadataDestinationDoesNotSayHowToAllowIt(
-	t *testing.T,
-) {
-	t.Parallel()
-
-	env := setupSourceTest(t)
-
-	metadataURLs := map[string]string{
-		"link-local": "http://169.254.169.254/latest/meta-data/",
-		"wireserver": "http://168.63.129.16/?comp=versions",
-	}
-
-	for name, metadataURL := range metadataURLs {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			webhook := seedWebhookWithRetention(t, env.db, 30)
-
-			form := url.Values{}
-			form.Set("name", "metadata")
-			form.Set("type", string(database.TargetTypeHTTP))
-			form.Set("url", metadataURL)
-
-			w := serveTarget(
-				env, http.MethodPost,
-				"/source/"+webhook.ID+"/targets", form,
-			)
-			assert.Equal(t, http.StatusBadRequest, w.Code)
-			assert.NotContains(
-				t, w.Body.String(), privateRefusalHint,
-			)
-		})
-	}
-}
@@ -140,7 +140,7 @@ func (n *noopEvictor) EvictWebhook(string) {}
 // and the database, exactly as internal/handlers builds them.
 //
 // One application per test function, not per case: every start that
-// finds no account seeds one at 64 MB of Argon2id, and this package's
+// finds no account seeds one with an Argon2id hash, and this package's
 // budget is not the place to spend that repeatedly.
 func newServerApp(
 	t *testing.T, dir string,
@@ -22,6 +22,11 @@
 # The one figure above 90s is GOMAXPROCS 1, a synthetic core floor rather than
 # a condition CI runs under. If a CPU-limited runner ever puts a real run near
 # 67s, that is the datum to revisit the org figure with.
+#
+# -p 4 -parallel 8 keep the run under 2 GB of memory: at most four test
+# binaries build or run at once, each with at most eight parallel tests. Under
+# -race every test binary and every link costs a few hundred MB, so the
+# defaults (one per core) add up to several GB on a many-core host.
 set -eu

 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -29,7 +34,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 main() {
    cd "$ROOT"
    "$ROOT/script/assets"
-    go test -v -race -timeout 90s ./...
+    go test -v -race -p 4 -parallel 8 -timeout 90s ./...
 }

 main "$@"