Keep make test under 2 GB of memory (closes #344)

Every test that starts a database hashed the bootstrap admin password
with Argon2id at 64 MB, which under -race holds about 150 MB per hash,
across dozens of parallel tests. HashPassword now hashes at 1 MB when
testing.Testing() reports a test binary, so every test package gets
the lower cost and a binary built by go build never does. One test
still hashes and verifies through HashPassword at the shipped
parameters, which are unchanged.

script/test also runs at most four packages and eight parallel tests
at once: unbounded, a many-core host linked and ran every test binary
together.

Model: opus-5-5

README.md

@@ -157,6 +157,21 @@ public cloud metadata addresses: currently only `168.63.129.16`, Azure's
 WireServer, which serves an Azure VM its credentials. Because it is a
 public address, listing it in `ALLOWED_EGRESS_CIDRS` reopens it.
 
+That is all the default blocklist covers: the IPv4 private and reserved
+ranges; of IPv6, only loopback (`::1`), unique local addresses
+(`fc00::/7`) and link-local addresses (`fe80::/10`); and certain public
+addresses. A public address belongs on the default blocklist only if it
+hands credentials, user data or bootstrap material to whatever can reach
+it, without the caller presenting anything. A provider's other public
+addresses are not refused. IBM Cloud, for example, serves its package
+mirrors, time servers and object storage on `161.26.0.0/16`, and the
+private endpoints of its own cloud services on `166.8.0.0/14`. Neither
+range hands out credentials that way: the token service among those
+endpoints issues a token only in exchange for something the caller
+presents, such as an API key. Reaching these services can be a
+legitimate delivery, and every cloud has some, so a partial list would
+promise coverage it does not give.
+
 That default is also inconvenient for the thing webhooker is mostly
 for: taking a public webhook and forwarding it to something on your own
 network. A container on the same Docker network, a box on `10.x`, a

internal/database/export_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	"testing"
 	"time"
 
 	"go.uber.org/fx"
@@ -79,3 +80,14 @@ func (d *Database) ExportSetBannerOut(w io.Writer) {
 func DummyPasswordHashForTest() string {
 	return dummyPasswordHash()
 }
+
+// HashAtShippedCostForTest makes HashPassword hash at the shipped
+// memory cost until t ends. t must not run in parallel with other
+// tests, which would hash at that cost alongside it.
+func HashAtShippedCostForTest(t *testing.T) {
+	t.Helper()
+
+	hashAtShippedCostInTest = true
+
+	t.Cleanup(func() { hashAtShippedCostInTest = false })
+}

internal/database/password.go

@@ -9,6 +9,7 @@ import (
 	"math/big"
 	"strings"
 	"sync"
+	"testing"
 
 	"golang.org/x/crypto/argon2"
 )
@@ -63,10 +64,30 @@ func DefaultPasswordConfig() *PasswordConfig {
 	}
 }
 
-// HashPassword generates an Argon2id hash of the password
+// testArgon2Memory is the Argon2id memory cost, in KiB, that a test
+// binary hashes with: 1 MB instead of the shipped 64 MB. Every test
+// that starts a database hashes the bootstrap admin password, dozens
+// of them run in parallel, and under the race detector each 64 MB hash
+// holds about 150 MB. VerifyPassword reads the cost from the hash it
+// checks, so verification follows.
+const testArgon2Memory = 1024
+
+// hashAtShippedCostInTest makes a test binary hash at the shipped
+// memory cost. Only TestHashPassword_ShippedParameters sets it.
+//
+//nolint:gochecknoglobals // set by one test, see above
+var hashAtShippedCostInTest bool
+
+// HashPassword generates an Argon2id hash of the password. A binary
+// built by go test hashes at testArgon2Memory; one built by go build
+// always hashes at the defaults.
 func HashPassword(password string) (string, error) {
 	config := DefaultPasswordConfig()
 
+	if testing.Testing() && !hashAtShippedCostInTest {
+		config.Memory = testArgon2Memory
+	}
+
 	// Generate a salt
 	salt := make([]byte, config.SaltLen)
 

internal/database/password_test.go

@@ -192,6 +192,39 @@ func TestHashPasswordUniqueness(t *testing.T) {
 	}
 }
 
+// TestHashPassword_ShippedParameters hashes and verifies through
+// HashPassword at the shipped Argon2id parameters. Every other test
+// hashes at the lower memory cost a test binary uses, so this is the
+// one that keeps production hashing covered. One hash and one
+// verification: each costs 64 MB.
+//
+//nolint:paralleltest // changes the hashing cost for the whole binary
+func TestHashPassword_ShippedParameters(t *testing.T) {
+	database.HashAtShippedCostForTest(t)
+
+	password := "correct horse battery staple"
+
+	hash, err := database.HashPassword(password)
+	if err != nil {
+		t.Fatalf("hashing with the shipped parameters: %v", err)
+	}
+
+	const shipped = "$argon2id$v=19$m=65536,t=1,p=4$"
+
+	if !strings.HasPrefix(hash, shipped) {
+		t.Errorf("hash = %q, want prefix %q", hash, shipped)
+	}
+
+	valid, err := database.VerifyPassword(password, hash)
+	if err != nil {
+		t.Fatalf("VerifyPassword() error = %v", err)
+	}
+
+	if !valid {
+		t.Error("VerifyPassword() returned false for correct password")
+	}
+}
+
 // TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
 // path. Login charges an unknown username a verification against a
 // dummy hash so that a nonexistent account is not answered in

internal/delivery/ssrf.go

@@ -17,10 +17,6 @@ const (
 	// dnsResolutionTimeout is the maximum time to wait for
 	// DNS resolution during SSRF validation.
 	dnsResolutionTimeout = 5 * time.Second
-
-	// azureWireServer is Azure's WireServer, a public address that
-	// serves VM credentials.
-	azureWireServer = "168.63.129.16"
 )
 
 // Sentinel errors for SSRF validation.
@@ -29,14 +25,8 @@ var (
 	errNoIPs      = errors.New(
 		"hostname resolved to no IP addresses",
 	)
-	// ErrBlockedIP reports a private or reserved address the
-	// default blocklist refuses, one that ALLOWED_EGRESS_CIDRS
-	// can open.
-	ErrBlockedIP = errors.New(
-		"blocked private or reserved address",
-	)
-	errBlockedWireServer = errors.New(
-		"blocked cloud metadata address",
+	errBlockedIP = errors.New(
+		"blocked private, reserved or cloud metadata address",
 	)
 	errBlockedMetadata = errors.New(
 		"blocked link-local or cloud instance metadata " +
@@ -53,6 +43,13 @@ var (
 // permit specific blocks out of this set with
 // ALLOWED_EGRESS_CIDRS; see Guard.
 //
+// A public address belongs on the default blocklist only if it
+// hands credentials, user data or bootstrap material to whatever
+// can reach it, without the caller presenting anything. A
+// provider's other public addresses are not refused, since
+// reaching them can be legitimate and no list of them could be
+// complete.
+//
 //nolint:gochecknoglobals // package-level network list is appropriate here
 var blockedNetworks []*net.IPNet
 
@@ -133,7 +130,8 @@ func init() {
 		"::1/128",
 		"fc00::/7",
 		"fe80::/10",
-		azureWireServer + "/32",
+		// Azure WireServer, a public address that serves VM credentials.
+		"168.63.129.16/32",
 	})
 
 	// Every entry is named. The set must not grow or shrink
@@ -347,17 +345,9 @@ func (g *Guard) checkIP(ip net.IP) error {
 		return nil
 	}
 
-	// WireServer is on the default blocklist but is a public
-	// address, so its refusal does not call it private or reserved.
-	if ip.Equal(net.ParseIP(azureWireServer)) {
-		return fmt.Errorf(
-			"target IP %s: %w", ip, errBlockedWireServer,
-		)
-	}
-
 	if isBlockedIP(ip) {
 		return fmt.Errorf(
-			"target IP %s: %w", ip, ErrBlockedIP,
+			"target IP %s: %w", ip, errBlockedIP,
 		)
 	}
 

internal/handlers/source_management.go

@@ -1577,22 +1577,11 @@ func (h *Handlers) validateTargetURL(
 			"url", delivery.MaskURL(targetURL),
 			"error", err,
 		)
-
-		msg := "Invalid target URL: " + err.Error()
-
-		// Only a private or reserved address's refusal says how
-		// to allow it. Metadata refusals never do: link-local and
-		// the other unconditional metadata addresses cannot be
-		// opened, and Azure's WireServer, which listing does
-		// open, hands out VM credentials.
-		if errors.Is(err, delivery.ErrBlockedIP) {
-			msg += ". Private and reserved addresses are refused " +
-				"by default; the server's ALLOWED_EGRESS_CIDRS " +
-				"setting allows named networks (see \"Allowing " +
-				"egress to your own network\" in the README)."
-		}
-
-		http.Error(w, msg, http.StatusBadRequest)
+		http.Error(
+			w,
+			"Invalid target URL: "+err.Error(),
+			http.StatusBadRequest,
+		)
 
 		return err
 	}

internal/handlers/target_private_refusal_test.go

@@ -1,116 +0,0 @@
-package handlers_test
-
-import (
-	"net/http"
-	"net/url"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/webhooker/internal/database"
-)
-
-// privateRefusalHint is the sentence that tells an operator a private
-// destination is refused on purpose, and how to allow one.
-const privateRefusalHint = "Private and reserved addresses are " +
-	"refused by default; the server's ALLOWED_EGRESS_CIDRS setting " +
-	"allows named networks (see \"Allowing egress to your own " +
-	"network\" in the README)."
-
-// TestTargetRefusal_PrivateDestinationSaysHowToAllowIt covers both
-// target types that take a URL, on add and on edit.
-func TestTargetRefusal_PrivateDestinationSaysHowToAllowIt(
-	t *testing.T,
-) {
-	t.Parallel()
-
-	env := setupSourceTest(t)
-
-	targetTypes := []database.TargetType{
-		database.TargetTypeHTTP,
-		database.TargetTypeSlack,
-	}
-
-	for _, targetType := range targetTypes {
-		t.Run(string(targetType), func(t *testing.T) {
-			t.Parallel()
-
-			webhook := seedWebhookWithRetention(t, env.db, 30)
-			targetsPath := "/source/" + webhook.ID + "/targets"
-
-			form := url.Values{}
-			form.Set("name", "private")
-			form.Set("type", string(targetType))
-			form.Set("url", editBlockedURL)
-
-			added := serveTarget(
-				env, http.MethodPost, targetsPath, form,
-			)
-			assert.Equal(t, http.StatusBadRequest, added.Code)
-			assert.Contains(
-				t, added.Body.String(), privateRefusalHint,
-			)
-
-			form.Set("url", editOriginalURL)
-
-			created := serveTarget(
-				env, http.MethodPost, targetsPath, form,
-			)
-			require.Equal(
-				t, http.StatusSeeOther, created.Code,
-				created.Body.String(),
-			)
-
-			targets := targetsForWebhook(t, env.db, webhook.ID)
-			require.Len(t, targets, 1)
-
-			form.Set("url", editBlockedURL)
-
-			edited := submitTargetEdit(
-				env, webhook.ID, targets[0].ID, form,
-			)
-			assert.Equal(t, http.StatusBadRequest, edited.Code)
-			assert.Contains(
-				t, edited.Body.String(), privateRefusalHint,
-			)
-		})
-	}
-}
-
-// TestTargetRefusal_MetadataDestinationDoesNotSayHowToAllowIt: no
-// setting opens a link-local address, and Azure's WireServer hands out
-// VM credentials, so neither refusal points at the setting.
-func TestTargetRefusal_MetadataDestinationDoesNotSayHowToAllowIt(
-	t *testing.T,
-) {
-	t.Parallel()
-
-	env := setupSourceTest(t)
-
-	metadataURLs := map[string]string{
-		"link-local": "http://169.254.169.254/latest/meta-data/",
-		"wireserver": "http://168.63.129.16/?comp=versions",
-	}
-
-	for name, metadataURL := range metadataURLs {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			webhook := seedWebhookWithRetention(t, env.db, 30)
-
-			form := url.Values{}
-			form.Set("name", "metadata")
-			form.Set("type", string(database.TargetTypeHTTP))
-			form.Set("url", metadataURL)
-
-			w := serveTarget(
-				env, http.MethodPost,
-				"/source/"+webhook.ID+"/targets", form,
-			)
-			assert.Equal(t, http.StatusBadRequest, w.Code)
-			assert.NotContains(
-				t, w.Body.String(), privateRefusalHint,
-			)
-		})
-	}
-}

internal/resetpw/resetpw_test.go

@@ -140,7 +140,7 @@ func (n *noopEvictor) EvictWebhook(string) {}
 // and the database, exactly as internal/handlers builds them.
 //
 // One application per test function, not per case: every start that
-// finds no account seeds one at 64 MB of Argon2id, and this package's
+// finds no account seeds one with an Argon2id hash, and this package's
 // budget is not the place to spend that repeatedly.
 func newServerApp(
 	t *testing.T, dir string,

script/test

@@ -22,6 +22,11 @@
 # The one figure above 90s is GOMAXPROCS 1, a synthetic core floor rather than
 # a condition CI runs under. If a CPU-limited runner ever puts a real run near
 # 67s, that is the datum to revisit the org figure with.
+#
+# -p 4 -parallel 8 keep the run under 2 GB of memory: at most four test
+# binaries build or run at once, each with at most eight parallel tests. Under
+# -race every test binary and every link costs a few hundred MB, so the
+# defaults (one per core) add up to several GB on a many-core host.
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -29,7 +34,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 main() {
     cd "$ROOT"
     "$ROOT/script/assets"
-    go test -v -race -timeout 90s ./...
+    go test -v -race -p 4 -parallel 8 -timeout 90s ./...
 }
 
 main "$@"