Health check answers 503 while report files cannot be written (closes #142)

The health check answered 200 "status":"ok" whatever happened, so a
container whose report files could not be written stayed healthy while
every report was refused with 500. The report buffer now says whether
its last write of a report file failed. While it has, the health
check's status is "error" and the handler answers 503 with the same
fields, which fails the image's HEALTHCHECK. At start no write has
failed, and the first write that succeeds brings back 200. A test
removes DATA_DIR to make a write fail and checks the 503, then the 200
once a write succeeds again.

Model: opus-5-5

internal/handlers/healthcheck.go

@@ -3,9 +3,17 @@ package handlers
 import "net/http"
 
 // HandleHealthCheck returns a handler for the health check
-// endpoint.
+// endpoint. It answers 200 while the status is "ok" and 503 otherwise,
+// so that the image's HEALTHCHECK fails.
 func (s *Handlers) HandleHealthCheck() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		s.respondJSON(w, r, s.hc.Healthcheck(), http.StatusOK)
+		resp := s.hc.Healthcheck()
+
+		code := http.StatusOK
+		if resp.Status != "ok" {
+			code = http.StatusServiceUnavailable
+		}
+
+		s.respondJSON(w, r, resp, code)
 	}
 }

internal/handlers/healthcheck_test.go

@@ -9,17 +9,19 @@ import (
 	"testing"
 	"time"
 
+	"sneak.berlin/go/netwatch/internal/config"
 	"sneak.berlin/go/netwatch/internal/globals"
 	"sneak.berlin/go/netwatch/internal/handlers"
 	"sneak.berlin/go/netwatch/internal/healthcheck"
 	"sneak.berlin/go/netwatch/internal/logger"
+	"sneak.berlin/go/netwatch/internal/reportbuf"
 
 	"go.uber.org/fx/fxtest"
 )
 
 // newStartedHandlers builds Handlers with a real health check for the
-// server named in g, and starts them, which records the time the
-// uptime counts from.
+// server named in g, and a report buffer in a temporary directory, and
+// starts them, which records the time the uptime counts from.
 func newStartedHandlers(t *testing.T, g *globals.Globals) *handlers.Handlers {
 	t.Helper()
 
@@ -30,14 +32,23 @@ func newStartedHandlers(t *testing.T, g *globals.Globals) *handlers.Handlers {
 		t.Fatalf("logger: %v", err)
 	}
 
+	buf, err := reportbuf.New(lc, reportbuf.Params{
+		Config: &config.Config{DataDir: t.TempDir(), DataDirMaxBytes: 1 << 20},
+		Logger: log,
+	})
+	if err != nil {
+		t.Fatalf("report buffer: %v", err)
+	}
+
 	hc, err := healthcheck.New(lc,
-		healthcheck.Params{Globals: g, Logger: log})
+		healthcheck.Params{Buffer: buf, Globals: g, Logger: log})
 	if err != nil {
 		t.Fatalf("health check: %v", err)
 	}
 
-	h, err := handlers.New(lc,
-		handlers.Params{Globals: g, Healthcheck: hc, Logger: log})
+	h, err := handlers.New(lc, handlers.Params{
+		Buffer: buf, Globals: g, Healthcheck: hc, Logger: log,
+	})
 	if err != nil {
 		t.Fatalf("handlers: %v", err)
 	}
@@ -48,10 +59,12 @@ func newStartedHandlers(t *testing.T, g *globals.Globals) *handlers.Handlers {
 	return h
 }
 
-// TestHandleHealthCheck checks the health check's answer: 200, a JSON
-// content type, and a JSON object with exactly the fields of
-// healthcheck.HealthcheckResponse, carrying this server's name and
-// version and an uptime counted from its start.
+// TestHandleHealthCheck checks the health check's answer at start, when
+// no write of a report file has failed: 200, a JSON content type, and a
+// JSON object with exactly the fields of healthcheck.HealthcheckResponse,
+// carrying "status":"ok", this server's name and version and an uptime
+// counted from its start. The answer while writes fail is checked in
+// internal/reportbuf, whose tests can make a write fail.
 func TestHandleHealthCheck(t *testing.T) {
 	t.Parallel()
 

internal/healthcheck/healthcheck.go

@@ -10,6 +10,7 @@ import (
 	"sneak.berlin/go/netwatch/internal/config"
 	"sneak.berlin/go/netwatch/internal/globals"
 	"sneak.berlin/go/netwatch/internal/logger"
+	"sneak.berlin/go/netwatch/internal/reportbuf"
 
 	"go.uber.org/fx"
 )
@@ -18,6 +19,7 @@ import (
 type Params struct {
 	fx.In
 
+	Buffer  *reportbuf.Buffer
 	Config  *config.Config
 	Globals *globals.Globals
 	Logger  *logger.Logger
@@ -68,12 +70,18 @@ func New(
 	return s, nil
 }
 
-// Healthcheck returns the current health status of the application.
+// Healthcheck returns the current health status of the application:
+// "ok", or "error" while report files cannot be written.
 func (s *Healthcheck) Healthcheck() *HealthcheckResponse {
+	status := "ok"
+	if s.params.Buffer.LastWriteFailed() {
+		status = "error"
+	}
+
 	return &HealthcheckResponse{
 		Appname:       s.params.Globals.Appname,
 		Now:           time.Now().UTC().Format(time.RFC3339Nano),
-		Status:        "ok",
+		Status:        status,
 		UptimeHuman:   s.uptime().String(),
 		UptimeSeconds: int64(s.uptime().Seconds()),
 		Version:       s.params.Globals.Version,

internal/reportbuf/reportbuf.go

@@ -255,6 +255,16 @@ func (b *Buffer) Append(v any) error {
 	return nil
 }
 
+// LastWriteFailed reports whether the last write of a report file to
+// finish failed: false at start until a write fails, and false again
+// once a write succeeds. While it is true, Append refuses reports.
+func (b *Buffer) LastWriteFailed() bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	return b.writeErr != nil
+}
+
 // deleteOldestFiles deletes report files, oldest first, until n more
 // bytes fit under maxBytes. It deletes none when the reports waiting
 // to be written leave no room for n even with every file gone, since

internal/reportbuf/reportbuf_test.go

@@ -1074,6 +1074,51 @@ func TestReportsRefusedWhileWritesFail(t *testing.T) {
 	}
 }
 
+// TestHealthCheckFailsWhileWritesFail asks the health check handler
+// for its answer around a failed write. It answers 200 with
+// "status":"ok" at start, 503 with "status":"error" from the failed
+// write until a write succeeds, and 200 with "status":"ok" again after.
+func TestHealthCheckFailsWhileWritesFail(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("DATA_DIR", dir)
+
+	buf, h := startBufferWithHandlers(t)
+
+	checkHealthCheck(t, h, "at start", http.StatusOK, "ok")
+
+	// A flush writes a file only when a report is waiting.
+	err := buf.Append(map[string]string{"id": "kept"})
+	if err != nil {
+		t.Fatalf("append: %v", err)
+	}
+
+	// With the data directory gone, the write has nowhere to go.
+	err = os.Remove(dir)
+	if err != nil {
+		t.Fatalf("remove data dir: %v", err)
+	}
+
+	err = buf.Flush()
+	if err == nil {
+		t.Fatal("flush succeeded with the data directory gone")
+	}
+
+	checkHealthCheck(t, h, "after the failed write",
+		http.StatusServiceUnavailable, "error")
+
+	err = os.Mkdir(dir, 0o750)
+	if err != nil {
+		t.Fatalf("make data dir: %v", err)
+	}
+
+	err = buf.Flush()
+	if err != nil {
+		t.Fatalf("flush with the data directory back: %v", err)
+	}
+
+	checkHealthCheck(t, h, "after a write succeeded", http.StatusOK, "ok")
+}
+
 // startBufferWithHandlers starts a Buffer and the Handlers that store
 // reports in it through fx, as main does, with the DATA_DIR the calling
 // test has set.
@@ -1121,6 +1166,39 @@ func postReport(t *testing.T, h *handlers.Handlers, clientID string) int {
 	return rec.Code
 }
 
+// checkHealthCheck asks the health check handler for its answer, and
+// fails the test, saying when it asked, unless the answer has status
+// code wantCode and a "status" field of wantStatus.
+func checkHealthCheck(
+	t *testing.T,
+	h *handlers.Handlers,
+	when string,
+	wantCode int,
+	wantStatus string,
+) {
+	t.Helper()
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(t.Context(),
+		http.MethodGet, "/.well-known/healthcheck", http.NoBody)
+	h.HandleHealthCheck().ServeHTTP(rec, req)
+
+	var body struct {
+		Status string `json:"status"`
+	}
+
+	err := json.Unmarshal(rec.Body.Bytes(), &body)
+	if err != nil {
+		t.Fatalf("health check %s: body not JSON: %v (%q)",
+			when, err, rec.Body.String())
+	}
+
+	if rec.Code != wantCode || body.Status != wantStatus {
+		t.Fatalf("health check %s: %d with status %q, want %d with %q",
+			when, rec.Code, body.Status, wantCode, wantStatus)
+	}
+}
+
 // TestConcurrentAppendsStopAtCap appends from many goroutines at once
 // with room for exactly roomFor reports: exactly that many must be
 // taken, which holds only if Append checks and counts each report