Stopping the server waits for report file writes under way (closes #122)

The report buffer's stop now waits, through a sync.WaitGroup, for
flushLoop and for the write Append starts at 10 MiB before its final
flush. It used to return at once, so the process exiting cut such a
write off and its reports were lost.

A write under way that fails puts its reports back in the buffer, so
the final flush writes them; the stop fails only if that flush fails
too.

Tests hold the periodic write and the one at 10 MiB open in a synctest
bubble and find the stop waiting until each is released; a third makes
the held write fail and finds its report written by the final flush.

Model: opus-5-5

Dockerfile

@@ -5,11 +5,11 @@
 #
 # The lint and test phases are the gates: `make lint` (script/lint)
 # builds the lint stage alone and `make test` (script/test) the test
-# stage alone, and the builder stage depends on both, so the image
-# cannot be built unless they pass. Each covers the frontend as well,
-# through a copy from a node stage. Inside them each tool is invoked
-# directly, never through make or script/, whose lint and test are
-# themselves docker builds.
+# stage alone, and the two build stages, frontend-build and builder,
+# depend on both, so the image cannot be built unless they pass. Each
+# phase covers the frontend as well, through a copy from a node stage.
+# Inside them each tool is invoked directly, never through make or
+# script/, whose lint and test are themselves docker builds.
 
 # Frontend lint stage: eslint with the rules in eslint.config.js. The
 # lint phase below runs it.
@@ -35,30 +35,26 @@ RUN golangci-lint run --config .golangci.yml ./...
 # phase run it.
 COPY --from=frontend-lint /app/yarn.lock /dev/null
 
-# Frontend stage: the unit tests in test/unit/, then the production
-# build into dist/, which the runtime stage serves. The test phase below
-# runs it. The tests print a dot each; if any fails, they run again with
-# every test listed, and the step fails even if that run passes.
-# NODE_OPTIONS chooses the reporter because yarn adds its arguments
-# after the test files, where node would take a reporter option for one
-# more file.
+# Frontend test stage: the unit tests in test/unit/. The test phase
+# below runs it, and the frontend-build stage starts from it. The tests
+# print a dot each; if any fails, they run again with every test listed,
+# and the step fails even if that run passes. NODE_OPTIONS chooses the
+# reporter because yarn adds its arguments after the test files, where
+# node would take a reporter option for one more file.
 # node:22-alpine as of 2026-02-22
-FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend
+FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend-test
 WORKDIR /app
 COPY package.json yarn.lock ./
 RUN yarn install --frozen-lockfile
-# vite.config.js reads the commit for the page's footer with git.
-RUN apk add --no-cache git
 COPY . .
 RUN NODE_OPTIONS=--test-reporter=dot timeout 90 yarn --silent run test || \
     { echo "--- Rerunning with every test listed for details ---"; \
       NODE_OPTIONS=--test-reporter=spec timeout 90 yarn --silent run test; \
       exit 1; }
-RUN yarn build
 
 # Test phase: the backend's tests with the race detector and coverage,
-# and the frontend's through the copy from the frontend stage at the
-# end. -race needs cgo and so a C compiler, which the Debian Go image
+# and the frontend's through the copy from the frontend-test stage at
+# the end. -race needs cgo and so a C compiler, which the Debian Go image
 # ships and the alpine one does not. -timeout 90s is a backstop above
 # the 60-second cap on the suite. The rerun with -v only shows details:
 # the step fails however it ends, because the first run already failed.
@@ -75,9 +71,21 @@ RUN --mount=type=tmpfs,target=/go/pkg/mod \
     go test -timeout 90s -race -cover ./... || \
     { echo "--- Rerunning with -v for details ---"; \
       go test -timeout 90s -race -v ./...; exit 1; }
-# Nothing is wanted from the frontend stage; the copy is what makes
+# Nothing is wanted from the frontend-test stage; the copy is what makes
 # this phase run its tests.
-COPY --from=frontend /app/yarn.lock /dev/null
+COPY --from=frontend-test /app/yarn.lock /dev/null
+
+# Frontend build stage: the production build into dist/, which the
+# runtime stage serves, on the yarn packages and source of the
+# frontend-test stage. Nothing is wanted from the two phases; the copies
+# are what make BuildKit build them first, so this stage cannot run
+# unless lint and test passed.
+FROM frontend-test AS frontend-build
+COPY --from=lint /src/go.sum /dev/null
+COPY --from=test /src/go.sum /dev/null
+# vite.config.js reads the commit for the page's footer with git.
+RUN apk add --no-cache git
+RUN yarn build
 
 # Backend build stage. Nothing is wanted from the two phases; the copies
 # are what make BuildKit build them first, so this stage cannot run
@@ -131,7 +139,7 @@ RUN addgroup -g 1000 -S netwatch && \
 RUN rm /etc/nginx/conf.d/default.conf
 COPY configs/nginx.conf /etc/nginx/templates/netwatch.conf.template
 COPY configs/security-headers.conf /etc/nginx/security-headers.conf
-COPY --from=frontend /app/dist /usr/share/nginx/html
+COPY --from=frontend-build /app/dist /usr/share/nginx/html
 COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server
 COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh
 

README.md

@@ -59,8 +59,9 @@ either one is broken. We provide:
 - `script/clean` — remove what `script/build` writes
 - `script/projectname` — print the project name (used for the Docker image tag)
 - `script/test` — build the `test` phase of `Dockerfile` without the cache: the
-  frontend's unit tests and production build in its `frontend` stage, and the
-  backend's Go tests with the race detector and coverage
+  frontend's unit tests in its `frontend-test` stage, and the backend's Go tests
+  with the race detector and coverage. The frontend's production build is left
+  to the image build, in the `frontend-build` stage
 - `script/lint` — build the `lint` phase of `Dockerfile` without the cache:
   eslint in its `frontend-lint` stage, and golangci-lint over the Go packages
 - `script/fmt` — format all files (writes): prettier over the JavaScript, CSS,
@@ -78,8 +79,8 @@ either one is broken. We provide:
 - `script/frontend-test` — run the unit tests in `test/unit/` on the host with
   Node's built-in test runner, through the `test` script in `package.json`, and
   if any fails, run them again listing every test, and fail; then the production
-  build. Each test run has a 90-second timeout. `make test` runs the same in
-  Docker
+  build. Each test run has a 90-second timeout. `make test` runs the same tests
+  in Docker, without the production build
 - `script/frontend-fmt` — format the JavaScript, CSS, HTML and Markdown with
   prettier (writes)
 - `script/frontend-fmt-check` — check prettier formatting (read-only)

TODO.md

@@ -21,6 +21,26 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of
 
 # Completed Steps
 
+- 2026-10-14: stopping the server waits for a report file write already under
+  way, the periodic flush's or the one `Append` starts when 10 MiB are waiting,
+  before the final flush
+  ([#122](https://git.eeqj.de/sneak/netwatch/issues/122)). It used to return at
+  once, and the process exiting cut the write off, losing its reports. A write
+  under way that fails puts its reports back in memory, so the final flush
+  writes them; the stop fails, and the process exits non-zero, only if that
+  flush fails too. Two tests hold the periodic write and the one at 10 MiB open
+  and find the stop waiting until each is released; a third makes the held write
+  fail and finds its report written by the final flush. They run in a
+  `testing/synctest` bubble, so the periodic flush comes without waiting a
+  minute
+- 2026-10-14: `make test` no longer runs the frontend's production build, which
+  the tests do not need ([#118](https://git.eeqj.de/sneak/netwatch/issues/118)).
+  `yarn build` moved out of the stage that runs the frontend's unit tests, now
+  named `frontend-test`, into a new `frontend-build` stage, which starts from it
+  and, like the `builder` stage, waits on the lint and test phases, so the image
+  still cannot be built unless they pass. The runtime stage copies `dist/` from
+  it, and the `git` that `vite.config.js` reads the commit with is installed
+  there. No build cache was added
 - 2026-10-14: opening the debug log no longer raises the measured latencies
   ([#131](https://git.eeqj.de/sneak/netwatch/issues/131)). With the panel open,
   each line written drew the panel again with every line before it, up to 1000,
@@ -503,3 +523,9 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of
 - A configurable host list (an environment variable or a config file)
 - Export of the latency history (CSV or JSON)
 - A notification when the health status changes to DEGRADED
+- `make test` within the 20-second target
+  ([#118](https://git.eeqj.de/sneak/netwatch/issues/118)): every run compiles
+  the Go standard library and every dependency with the race detector from
+  nothing. Whether test builds may keep Go's compiled packages between runs
+  waits on sneak's ruling in
+  [prompts#113](https://git.eeqj.de/sneak/prompts/issues/113)

internal/reportbuf/reportbuf.go

@@ -96,6 +96,10 @@ type Buffer struct {
 	// refuses reports, and the reports of that write wait in buf for
 	// the next flush.
 	writeErr error
+	// writers are the goroutines that write report files: flushLoop,
+	// and each one Append starts at flushSizeThreshold. OnStop waits
+	// for them before the final flush.
+	writers sync.WaitGroup
 }
 
 // New creates a Buffer and registers lifecycle hooks to
@@ -147,7 +151,7 @@ func New(
 
 			b.mu.Unlock()
 
-			go b.flushLoop()
+			b.writers.Go(b.flushLoop)
 
 			return nil
 		},
@@ -158,7 +162,14 @@ func New(
 			var err error
 
 			b.stopOnce.Do(func() {
+				// Closing done ends flushLoop after any flush it is
+				// running. Wait for it and for the writes Append
+				// started, which the process exiting would cut off;
+				// one that fails puts its reports back in buf, for the
+				// final flush to write.
 				close(b.done)
+				b.writers.Wait()
+
 				err = b.flushLocked()
 			})
 
@@ -211,12 +222,12 @@ func (b *Buffer) Append(v any) error {
 		data := b.drainBuf()
 		b.mu.Unlock()
 
-		go func() {
+		b.writers.Go(func() {
 			writeErr := b.writeFile(data)
 			if writeErr != nil {
 				b.log.Error("flush reports failed", "error", writeErr)
 			}
-		}()
+		})
 
 		return nil
 	}

internal/reportbuf/reportbuf_test.go

@@ -15,6 +15,7 @@ import (
 	"sync"
 	"sync/atomic"
 	"testing"
+	"testing/synctest"
 	"time"
 
 	"sneak.berlin/go/netwatch/internal/config"
@@ -107,9 +108,174 @@ func TestFailedFinalFlushFailsStop(t *testing.T) {
 	}
 }
 
-// startBuffer starts a Buffer through fx, as main does, with the
-// DATA_DIR and DATA_DIR_MAX_BYTES the calling test has set.
-func startBuffer(t *testing.T) *reportbuf.Buffer {
+// TestStopWaitsForPeriodicWrite holds open the write the periodic flush
+// starts, then stops the app. The stop must wait until the write is
+// released, and the report must then be in its file. The test runs in
+// a synctest bubble, whose clock moves on at once when every goroutine
+// in it is waiting, so the periodic flush does not take a minute.
+func TestStopWaitsForPeriodicWrite(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("DATA_DIR", dir)
+
+	synctest.Test(t, func(t *testing.T) {
+		buf, app := startApp(t)
+
+		created := make(chan struct{})
+		release := make(chan struct{})
+
+		buf.OnFileCreated(func(*os.File) {
+			created <- struct{}{}
+
+			<-release
+		})
+
+		err := buf.Append(map[string]string{"id": "periodic"})
+		if err != nil {
+			t.Fatalf("append report: %v", err)
+		}
+
+		<-created
+
+		err = stopWhileHeld(t, app, release)
+		if err != nil {
+			t.Fatalf("stop: %v", err)
+		}
+
+		files, err := readReportFiles(dir)
+		if err != nil {
+			t.Fatalf("read report files: %v", err)
+		}
+
+		want := `{"id":"periodic"}` + "\n"
+		if len(files) != 1 || files[0] != want {
+			t.Fatalf("report files = %q, want one holding %q", files, want)
+		}
+	})
+}
+
+// TestStopWaitsForWriteAtSizeThreshold holds open the write Append
+// starts when the buffer reaches FlushSizeThreshold, then stops the
+// app. The stop must wait until the write is released, and the reports
+// must then be in its file.
+func TestStopWaitsForWriteAtSizeThreshold(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("DATA_DIR", dir)
+
+	synctest.Test(t, func(t *testing.T) {
+		buf, app := startApp(t)
+
+		created := make(chan struct{})
+		release := make(chan struct{})
+
+		buf.OnFileCreated(func(*os.File) {
+			created <- struct{}{}
+
+			<-release
+		})
+
+		appended := appendToThreshold(t, buf)
+
+		<-created
+
+		err := stopWhileHeld(t, app, release)
+		if err != nil {
+			t.Fatalf("stop: %v", err)
+		}
+
+		files, err := readReportFiles(dir)
+		if err != nil {
+			t.Fatalf("read report files: %v", err)
+		}
+
+		if len(files) != 1 || files[0] != appended {
+			t.Fatalf("%d report files, want one holding the %d bytes "+
+				"appended", len(files), len(appended))
+		}
+	})
+}
+
+// TestFinalFlushWritesReportsOfFailedWrite holds open the write the
+// periodic flush starts, makes it fail, then stops the app. The stop
+// waits for the write, which puts its report back in the buffer, and
+// the final flush writes it, so the stop succeeds with nothing lost.
+func TestFinalFlushWritesReportsOfFailedWrite(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("DATA_DIR", dir)
+
+	synctest.Test(t, func(t *testing.T) {
+		buf, app := startApp(t)
+
+		created := make(chan *os.File)
+		release := make(chan struct{})
+
+		buf.OnFileCreated(func(f *os.File) {
+			created <- f
+
+			<-release
+		})
+
+		err := buf.Append(map[string]string{"id": "failed"})
+		if err != nil {
+			t.Fatalf("append report: %v", err)
+		}
+
+		held := <-created
+
+		// Closing the file under the write makes the write fail.
+		_ = held.Close()
+
+		// The final flush's write goes through.
+		buf.OnFileCreated(func(*os.File) {})
+
+		err = stopWhileHeld(t, app, release)
+		if err != nil {
+			t.Fatalf("stop: %v", err)
+		}
+
+		files, err := readReportFiles(dir)
+		if err != nil {
+			t.Fatalf("read report files: %v", err)
+		}
+
+		want := `{"id":"failed"}` + "\n"
+		if len(files) != 1 || files[0] != want {
+			t.Fatalf("report files = %q, want one holding %q", files, want)
+		}
+	})
+}
+
+// stopWhileHeld stops app while a report file write is held open until
+// release is closed, and fails the test if the stop returns before
+// that. It then closes release and returns the stop's error. It must
+// be called in a synctest bubble.
+func stopWhileHeld(
+	t *testing.T,
+	app *fxtest.App,
+	release chan<- struct{},
+) error {
+	t.Helper()
+
+	stopped := make(chan error, 1)
+
+	go func() { stopped <- app.Stop(t.Context()) }()
+
+	// Wait returns once every other goroutine in the bubble is waiting:
+	// the stop has got as far as it can while the write is held.
+	synctest.Wait()
+
+	if len(stopped) != 0 {
+		t.Error("stop returned while a report file write was held open")
+	}
+
+	close(release)
+
+	return <-stopped
+}
+
+// startApp starts a Buffer through fx, as main does, with the DATA_DIR
+// and DATA_DIR_MAX_BYTES the calling test has set, and returns it with
+// the app, for tests that stop the app themselves.
+func startApp(t *testing.T) (*reportbuf.Buffer, *fxtest.App) {
 	t.Helper()
 
 	var buf *reportbuf.Buffer
@@ -127,6 +293,16 @@ func startBuffer(t *testing.T) *reportbuf.Buffer {
 	app.RequireStart()
 	t.Cleanup(app.RequireStop)
 
+	return buf, app
+}
+
+// startBuffer starts a Buffer through fx, as main does, with the
+// DATA_DIR and DATA_DIR_MAX_BYTES the calling test has set.
+func startBuffer(t *testing.T) *reportbuf.Buffer {
+	t.Helper()
+
+	buf, _ := startApp(t)
+
 	return buf
 }
 
@@ -892,6 +1068,32 @@ func TestFlushAtSizeThreshold(t *testing.T) {
 
 	buf := startBuffer(t)
 
+	appended := appendToThreshold(t, buf)
+
+	// Append writes the file in the background, so wait for it.
+	deadline := time.Now().Add(10 * time.Second)
+
+	for {
+		files, err := readReportFiles(dir)
+		if err == nil && len(files) == 1 && files[0] == appended {
+			return
+		}
+
+		if time.Now().After(deadline) {
+			t.Fatalf("%d report files (error: %v), want one holding the "+
+				"%d bytes appended", len(files), err, len(appended))
+		}
+
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+
+// appendToThreshold appends reports to buf until it holds
+// FlushSizeThreshold bytes, so that the last append starts writing them
+// to a report file, and returns the lines appended.
+func appendToThreshold(t *testing.T, buf *reportbuf.Buffer) string {
+	t.Helper()
+
 	// Large reports, so the threshold takes a few hundred appends.
 	pad := strings.Repeat("a", 64<<10)
 
@@ -914,22 +1116,7 @@ func TestFlushAtSizeThreshold(t *testing.T) {
 		appended.WriteByte('\n')
 	}
 
-	// Append writes the file in the background, so wait for it.
-	deadline := time.Now().Add(10 * time.Second)
-
-	for {
-		files, err := readReportFiles(dir)
-		if err == nil && len(files) == 1 && files[0] == appended.String() {
-			return
-		}
-
-		if time.Now().After(deadline) {
-			t.Fatalf("%d report files (error: %v), want one holding the "+
-				"%d bytes appended", len(files), err, appended.Len())
-		}
-
-		time.Sleep(10 * time.Millisecond)
-	}
+	return appended.String()
 }
 
 // reportFilesBytes returns the total size of the report files in dir.

script/frontend-test

@@ -2,9 +2,10 @@
 # script/frontend-test: run the frontend test suite on the host: the
 # unit tests in test/unit/, through the test script in package.json,
 # then the production build, which fails on broken code. make test runs
-# the same in the frontend stage of Dockerfile. The tests print a dot
-# each; if any fails, they run again with every test listed, and the
-# script fails even if that run passes. NODE_OPTIONS chooses the
+# the same tests in the frontend-test stage of Dockerfile, and the image
+# build runs the production build in its frontend-build stage. The tests
+# print a dot each; if any fails, they run again with every test listed,
+# and the script fails even if that run passes. NODE_OPTIONS chooses the
 # reporter because yarn adds its arguments after the test files, where
 # node would take a reporter option for one more file.
 set -eu