Deploy main 5e572ce to prod (#198)

main's tree, with next merged on 26 October, as one commit onto prod, on sneak's order.

Model: opus-5-5
Co-authored-by: clawbot <clawbot@noreply.example.org>

internal/reportbuf/datadir.go

@@ -0,0 +1,180 @@
+package reportbuf
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"syscall"
+)
+
+// ErrDataDirOutsideVolume is returned by PrepareDataDir for a DATA_DIR
+// that is not the volume or a path below it, written in full.
+var ErrDataDirOutsideVolume = errors.New(
+	"must be /data or a path below it, with no '.', '..' or extra '/'")
+
+// PrepareDataDir gets dir, the server's DATA_DIR, ready for owner, the
+// user the server runs as, so that a host directory mounted at volume,
+// /data in the image, needs no preparing: it creates dir, gives volume
+// and the directories, regular files and symbolic links in it to
+// owner, and gives volume and dir the mode the server gives a directory
+// it creates. It leaves a file with a second name (hard link) as it
+// is, as that name could be outside volume, and logs its path; it
+// leaves a device, named pipe or socket as it is too (see chownFile).
+// dir must be volume or a path below it, with no '.', '..', empty part
+// or '/' at the end. On any system but Linux, the system of the image, it
+// fails at the first file in volume that is not a directory (see
+// chownFile).
+//
+// bin/entrypoint.sh runs this as root, which would follow a symbolic
+// link anywhere, so every step goes through an os.Root opened on
+// volume: it follows a link only when it is written as a relative
+// path that stays inside volume, and refuses any other. A process on
+// the host can swap a link onto a path in volume at any moment while
+// this runs. Even then, the os.Root checks each link as it reaches
+// it. MkdirAll creates each directory inside a parent it already has
+// open, never following a link at the name it creates, and follows a
+// link on the path only as the os.Root allows, so a relative link
+// inside volume can lead it to create directories elsewhere inside
+// volume. The owners and modes are set on files already opened (see
+// chownAll and chmodDir), so the most that process can do is make a
+// step fail or wait, or act on something else inside volume.
+func PrepareDataDir(volume, dir string, owner *user.User) error {
+	// rel is dir as a path from volume; IsLocal is false for one that
+	// leads out of it.
+	rel, err := filepath.Rel(volume, dir)
+	if err != nil || dir != filepath.Clean(dir) || !filepath.IsLocal(rel) {
+		return ErrDataDirOutsideVolume
+	}
+
+	uid, err := strconv.Atoi(owner.Uid)
+	if err != nil {
+		return err
+	}
+
+	gid, err := strconv.Atoi(owner.Gid)
+	if err != nil {
+		return err
+	}
+
+	root, err := os.OpenRoot(volume)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = root.Close() }()
+
+	err = root.MkdirAll(rel, dirPerms)
+	if err != nil {
+		return err
+	}
+
+	err = chownAll(root, ".", uid, gid)
+	if err != nil {
+		return err
+	}
+
+	err = chmodDir(root, ".")
+	if err != nil {
+		return err
+	}
+
+	return chmodDir(root, rel)
+}
+
+// chownAll gives name, a directory inside root, and the directories,
+// regular files and symbolic links in it to uid and gid, but not the
+// files chownFile leaves. It opens each through root and gives the
+// open file to them, as chmodDir sets a mode on the open directory.
+// Root.Lchown would not hold: it acts on whatever is at the name by
+// then, and a hard link swapped in there shares its owner with a file
+// that could be outside root. A symbolic link is given to them itself,
+// never what it points to, because in a directory that anyone can
+// write to and that has the sticky bit set, as /tmp has, the kernel
+// follows a link only for the link's owner or when the link and the
+// directory have the same owner. chownAll goes into an entry only when
+// the read found a directory there, so it follows no link it finds;
+// one swapped in afterwards for a directory is followed only as the
+// os.Root allows, and one swapped in for any other file is not
+// followed at all (see chownFile).
+func chownAll(root *os.Root, name string, uid, gid int) error {
+	entries, err := chownDir(root, name, uid, gid)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range entries {
+		entryName := filepath.Join(name, entry.Name())
+		if entry.IsDir() {
+			err = chownAll(root, entryName, uid, gid)
+		} else {
+			err = chownFile(root, entryName, uid, gid)
+		}
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// chownDir gives name, a directory inside root, to uid and gid, and
+// returns what it holds. Like chmodDir, it opens name through root and
+// refuses anything but a directory, which has no second name (hard
+// link). It reads the open directory, not root.FS(), which refuses a
+// name that is not valid UTF-8.
+func chownDir(root *os.Root, name string, uid, gid int) ([]fs.DirEntry, error) {
+	dir, err := root.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() { _ = dir.Close() }()
+
+	info, err := dir.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	if !info.IsDir() {
+		return nil, &fs.PathError{Op: "chown", Path: name, Err: syscall.ENOTDIR}
+	}
+
+	err = dir.Chown(uid, gid)
+	if err != nil {
+		return nil, err
+	}
+
+	return dir.ReadDir(-1)
+}
+
+// chmodDir gives name, a directory inside root, the mode the server
+// gives a directory it creates. Root.Chmod would not hold: on Linux it
+// checks that name is not a symbolic link, then sets the mode by name,
+// following a link swapped in between. So chmodDir opens name through
+// root and sets the mode on the open directory. It refuses anything
+// but a directory: a directory has no second name (hard link), so the
+// one opened is inside root, where any other file could be a hard link
+// to one outside.
+func chmodDir(root *os.Root, name string) error {
+	dir, err := root.Open(name)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = dir.Close() }()
+
+	info, err := dir.Stat()
+	if err != nil {
+		return err
+	}
+
+	if !info.IsDir() {
+		return &fs.PathError{Op: "chmod", Path: name, Err: syscall.ENOTDIR}
+	}
+
+	return dir.Chmod(dirPerms)
+}

internal/reportbuf/datadir_linux.go

@@ -0,0 +1,65 @@
+//go:build linux
+
+package reportbuf
+
+import (
+	"io/fs"
+	"log/slog"
+	"os"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// chownFile gives name, a file inside root that is not a directory, to
+// uid and gid if it is a regular file or symbolic link with one name.
+// It leaves, and logs, a file of any kind with a second name (hard
+// link): all the names of a file share its owner, and the other could
+// be outside root. It leaves any other file with one name, such as a
+// device, named pipe or socket: the server needs to own none of them.
+// It opens name through root with O_PATH, which reads nothing from the
+// file, so opening a device or named pipe neither waits nor acts on
+// it; os.Root adds O_NOFOLLOW, and the two together open a symbolic
+// link itself, never what it points to. It reads the kind of file and
+// its number of names from the open file and gives that file to them,
+// so a hard link swapped in at name is caught too.
+//
+// This file is built only on Linux: prepare-data-dir runs only in the
+// Linux image, and macOS, where the backend is also built for
+// development, has neither O_PATH nor AT_EMPTY_PATH. On any other
+// system chownFile returns an error (see datadir_other.go).
+func chownFile(root *os.Root, name string, uid, gid int) error {
+	file, err := root.OpenFile(name, unix.O_PATH, 0)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	info, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	stat, ok := info.Sys().(*syscall.Stat_t)
+	if !ok || stat.Nlink > 1 {
+		slog.Warn("owner not changed: file has a second name (hard link)",
+			"path", file.Name())
+
+		return nil
+	}
+
+	if !info.Mode().IsRegular() && info.Mode().Type() != fs.ModeSymlink {
+		return nil
+	}
+
+	// File.Chown would fail: fchown refuses a file opened with O_PATH.
+	// Fchownat with an empty name and AT_EMPTY_PATH acts on the open
+	// file instead.
+	err = unix.Fchownat(int(file.Fd()), "", uid, gid, unix.AT_EMPTY_PATH)
+	if err != nil {
+		return &fs.PathError{Op: "chown", Path: file.Name(), Err: err}
+	}
+
+	return nil
+}

internal/reportbuf/datadir_other.go

@@ -0,0 +1,21 @@
+//go:build !linux
+
+package reportbuf
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+)
+
+// chownFile returns an error on every system but Linux. Opening a file
+// that is not a directory without reading it or following it, and
+// giving it to a user on the open file, never by name, is built only
+// for Linux (see datadir_linux.go): prepare-data-dir runs only in the
+// Linux image, and macOS, where the backend is also built for
+// development, has neither O_PATH nor AT_EMPTY_PATH. This file lets
+// the package build elsewhere, such as on macOS, where a developer
+// runs make build and make run.
+func chownFile(_ *os.Root, name string, _, _ int) error {
+	return &fs.PathError{Op: "chown", Path: name, Err: errors.ErrUnsupported}
+}

internal/reportbuf/datadir_test.go

@@ -0,0 +1,339 @@
+package reportbuf_test
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"syscall"
+	"testing"
+
+	"sneak.berlin/go/netwatch/internal/reportbuf"
+)
+
+// reports is the last part of DATA_DIR in these tests, as in the
+// image's /data/reports.
+const reports = "reports"
+
+// currentUser is the user the test runs as, the only owner a test not
+// run as root can give files to.
+func currentUser() *user.User {
+	return &user.User{
+		Uid: strconv.Itoa(os.Getuid()),
+		Gid: strconv.Itoa(os.Getgid()),
+	}
+}
+
+// tempDirMode700 is a new directory in a t.TempDir with mode 0700, so
+// a test can tell that PrepareDataDir left its mode alone.
+func tempDirMode700(t *testing.T) string {
+	t.Helper()
+
+	dir := filepath.Join(t.TempDir(), "d")
+
+	err := os.Mkdir(dir, 0o700)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return dir
+}
+
+func requireMode(t *testing.T, path string, want fs.FileMode) {
+	t.Helper()
+
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if info.Mode() != want {
+		t.Errorf("%s: mode %v, want %v", path, info.Mode(), want)
+	}
+}
+
+func requireMissing(t *testing.T, path string) {
+	t.Helper()
+
+	_, err := os.Lstat(path)
+	if !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("%s: created, or Lstat failed: %v", path, err)
+	}
+}
+
+func requireOwner(t *testing.T, path string, uid, gid uint32) {
+	t.Helper()
+
+	info, err := os.Lstat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stat, _ := info.Sys().(*syscall.Stat_t)
+	if stat.Uid != uid || stat.Gid != gid {
+		t.Errorf("%s: owner %d:%d, want %d:%d", path, stat.Uid, stat.Gid,
+			uid, gid)
+	}
+}
+
+func TestPrepareDataDirCreatesDataDir(t *testing.T) {
+	t.Parallel()
+
+	volume := t.TempDir()
+	dir := filepath.Join(volume, "a", reports)
+
+	err := reportbuf.PrepareDataDir(volume, dir, currentUser())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	requireMode(t, volume, fs.ModeDir|0o750)
+	requireMode(t, dir, fs.ModeDir|0o750)
+}
+
+// TestPrepareDataDirSetsModeOfExistingDataDir: a DATA_DIR already on
+// the host with another mode gets the mode too, not only a new one.
+func TestPrepareDataDirSetsModeOfExistingDataDir(t *testing.T) {
+	t.Parallel()
+
+	volume := t.TempDir()
+	dir := filepath.Join(volume, reports)
+
+	err := os.Mkdir(dir, 0o700)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = reportbuf.PrepareDataDir(volume, dir, currentUser())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	requireMode(t, dir, fs.ModeDir|0o750)
+}
+
+func TestPrepareDataDirTakesTheVolumeItself(t *testing.T) {
+	t.Parallel()
+
+	volume := t.TempDir()
+
+	err := reportbuf.PrepareDataDir(volume, volume, currentUser())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	requireMode(t, volume, fs.ModeDir|0o750)
+}
+
+// TestPrepareDataDirRefusesDataDirOutsideVolume covers a DATA_DIR that
+// is relative, outside the volume, or not written in full.
+func TestPrepareDataDirRefusesDataDirOutsideVolume(t *testing.T) {
+	t.Parallel()
+
+	volume := tempDirMode700(t)
+	for _, dir := range []string{
+		reports, volume + "/../new", volume + "/", volume + "//" + reports,
+		volume + "/./" + reports, volume + "/" + reports + "/..", volume + "x",
+		"/etc",
+	} {
+		err := reportbuf.PrepareDataDir(volume, dir, currentUser())
+		if !errors.Is(err, reportbuf.ErrDataDirOutsideVolume) {
+			t.Errorf("%q: error = %v, want ErrDataDirOutsideVolume", dir, err)
+		}
+	}
+
+	requireMissing(t, filepath.Join(filepath.Dir(volume), "new"))
+	requireMode(t, volume, fs.ModeDir|0o700)
+}
+
+// TestPrepareDataDirRefusesLinkOutOfVolume puts a symbolic link to a
+// directory outside the volume on the path to DATA_DIR, written as a
+// full path and as one that climbs out with '..', and as DATA_DIR
+// itself, where the mode would be set through it.
+func TestPrepareDataDirRefusesLinkOutOfVolume(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name      string
+		climbsOut bool
+		link, dir string
+	}{
+		{"full path", false, "x", "x/reports"},
+		{"climbs out", true, "x", "x/reports"},
+		{"DATA_DIR itself", false, reports, reports},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			outside := tempDirMode700(t)
+			volume := t.TempDir()
+
+			climbOut, err := filepath.Rel(volume, outside)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			target := outside
+			if tc.climbsOut {
+				target = climbOut
+			}
+
+			err = os.Symlink(target, filepath.Join(volume, tc.link))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = reportbuf.PrepareDataDir(volume,
+				filepath.Join(volume, tc.dir), currentUser())
+			if err == nil {
+				t.Error("no error")
+			}
+
+			requireMissing(t, filepath.Join(outside, reports))
+			requireMode(t, outside, fs.ModeDir|0o700)
+		})
+	}
+}
+
+// TestPrepareDataDirRefusesDanglingLink: DATA_DIR is a symbolic link
+// to a name in the volume that does not exist, which is not created.
+func TestPrepareDataDirRefusesDanglingLink(t *testing.T) {
+	t.Parallel()
+
+	volume := t.TempDir()
+
+	err := os.Symlink("missing", filepath.Join(volume, reports))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = reportbuf.PrepareDataDir(volume, filepath.Join(volume, reports),
+		currentUser())
+	if err == nil {
+		t.Error("no error")
+	}
+
+	requireMissing(t, filepath.Join(volume, "missing"))
+}
+
+// TestPrepareDataDirTakesDirectoryNamedInLatin1: a host directory can
+// hold names that are not valid UTF-8, here "café" written in Latin-1.
+// A test not run as root can only check that PrepareDataDir goes into
+// such a directory and gives it, and what it holds, to the current
+// user.
+func TestPrepareDataDirTakesDirectoryNamedInLatin1(t *testing.T) {
+	t.Parallel()
+
+	volume := t.TempDir()
+	latin1 := filepath.Join(volume, "caf\xe9")
+
+	err := os.Mkdir(latin1, 0o700)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.WriteFile(filepath.Join(latin1, "f"), nil, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	owner := currentUser()
+
+	err = reportbuf.PrepareDataDir(volume, filepath.Join(volume, reports),
+		owner)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	uid, _ := strconv.ParseUint(owner.Uid, 10, 32)
+	gid, _ := strconv.ParseUint(owner.Gid, 10, 32)
+
+	requireOwner(t, latin1, uint32(uid), uint32(gid))
+	requireOwner(t, filepath.Join(latin1, "f"), uint32(uid), uint32(gid))
+}
+
+// TestPrepareDataDirGivesVolumeToOwner gives the directories, regular
+// files and symbolic links in the volume to a uid and gid that own
+// nothing, which only root can do. A symbolic link in the volume to a
+// directory outside it is given to them itself; what it points to is
+// left as it was. A hard link to a file outside the volume, one to a
+// symbolic link outside it and one to a named pipe outside it are left
+// as they were, and so is a named pipe in the volume.
+func TestPrepareDataDirGivesVolumeToOwner(t *testing.T) {
+	t.Parallel()
+
+	if os.Geteuid() != 0 {
+		t.Skip("only root can give files to another uid")
+	}
+
+	outside := t.TempDir()
+	volume := t.TempDir()
+	old := filepath.Join(volume, "old")
+
+	err := os.WriteFile(filepath.Join(outside, "f"), nil, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.Symlink("f", filepath.Join(outside, "link"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = syscall.Mkfifo(filepath.Join(outside, "fifo"), 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.Mkdir(old, 0o700)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.WriteFile(filepath.Join(old, "f"), nil, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.Symlink(outside, filepath.Join(old, "link"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = syscall.Mkfifo(filepath.Join(old, "fifo"), 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// os.Link does not follow a symbolic link: it gives the link itself
+	// a second name.
+	for _, name := range []string{"f", "link", "fifo"} {
+		err = os.Link(filepath.Join(outside, name),
+			filepath.Join(old, "hardlink-to-"+name))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	err = reportbuf.PrepareDataDir(volume, filepath.Join(volume, reports),
+		&user.User{Uid: "4242", Gid: "4343"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, path := range []string{
+		volume, filepath.Join(volume, reports), old, filepath.Join(old, "f"),
+		filepath.Join(old, "link"),
+	} {
+		requireOwner(t, path, 4242, 4343)
+	}
+
+	for _, path := range []string{
+		outside, filepath.Join(outside, "f"), filepath.Join(outside, "link"),
+		filepath.Join(outside, "fifo"), filepath.Join(old, "fifo"),
+	} {
+		requireOwner(t, path, 0, 0)
+	}
+}

internal/reportbuf/export_test.go

@@ -0,0 +1,29 @@
+package reportbuf
+
+import (
+	"os"
+	"time"
+)
+
+// FlushSizeThreshold exposes the buffer size at which Append starts
+// writing a report file to the external tests.
+const FlushSizeThreshold = flushSizeThreshold
+
+// Flush writes the buffered reports to a file now, as the periodic
+// flush does, so tests need not wait a minute for it.
+func (b *Buffer) Flush() error {
+	return b.flushLocked()
+}
+
+// StopClock makes every report file the buffer writes from now on
+// carry the timestamp at, as if all were written in one millisecond.
+func (b *Buffer) StopClock(at time.Time) {
+	b.now = func() time.Time { return at }
+}
+
+// OnFileCreated makes the buffer call fn with each report file it
+// writes from now on, once the file is created and before anything is
+// written to it.
+func (b *Buffer) OnFileCreated(fn func(f *os.File)) {
+	b.fileCreated = fn
+}

internal/reportbuf/reportbuf.go

@@ -0,0 +1,492 @@
+// Package reportbuf accumulates telemetry reports in memory
+// and periodically flushes them to zstd-compressed JSONL files,
+// deleting the oldest files to keep them under a size cap. Before the
+// server starts, PrepareDataDir gets the directory those files are
+// kept in, DATA_DIR, ready for the user the server runs as: it creates
+// it and gives it and the volume it is on to that user.
+package reportbuf
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"sneak.berlin/go/netwatch/internal/config"
+	"sneak.berlin/go/netwatch/internal/logger"
+
+	"github.com/klauspost/compress/zstd"
+	"go.uber.org/fx"
+)
+
+const (
+	flushSizeThreshold             = 10 << 20 // 10 MiB
+	flushInterval                  = 1 * time.Minute
+	defaultDataDir                 = "./data/reports"
+	dirPerms           fs.FileMode = 0o750
+	filePerms          fs.FileMode = 0o640
+
+	// Report files are named filePrefix + timestamp + "-" + number +
+	// fileSuffix; see writeFile.
+	filePrefix = "reports-"
+	fileSuffix = ".jsonl.zst"
+)
+
+// ErrFull is returned by Append when the reports waiting to be
+// written leave no room for the report under the configured maximum
+// size, however many report files are deleted.
+var ErrFull = errors.New("reports waiting to be written fill the size cap")
+
+// Params defines the dependencies for Buffer.
+type Params struct {
+	fx.In
+
+	Config *config.Config
+	Logger *logger.Logger
+}
+
+// reportFile is a report file that may be deleted to make room, with
+// the size it counts for in usedBytes.
+type reportFile struct {
+	name string
+	size int64
+}
+
+// Buffer accumulates JSON lines in memory and flushes them
+// to zstd-compressed files on disk.
+type Buffer struct {
+	buf     bytes.Buffer
+	dataDir string
+	done    chan struct{}
+	// fileCreated is called with each report file once it is
+	// created, before anything is written to it: it does nothing,
+	// except in tests that hold the write open or make it fail.
+	fileCreated func(f *os.File)
+	// files are the report files that may be deleted to make room,
+	// oldest first: those in dataDir at start, in name order, then
+	// each one this buffer writes, once it is complete. A file still
+	// being written is not among them. filesBytes is their total size.
+	files      []reportFile
+	filesBytes int64
+	log        *slog.Logger
+	maxBytes   int64
+	mu         sync.Mutex
+	// now is the clock report files are named by: time.Now, except
+	// in tests that need two flushes to share a timestamp.
+	now func() time.Time
+	// seq numbers the report files, so that two named in the same
+	// millisecond still get different names.
+	seq      atomic.Int64
+	stopOnce sync.Once
+	// thresholdWriteWaiting is set while a write Append started at
+	// flushSizeThreshold waits for writeMu; Append starts no other
+	// while it is set.
+	thresholdWriteWaiting atomic.Bool
+	// usedBytes is what Append checks against maxBytes: the size
+	// of the report files in dataDir, plus the reports waiting to
+	// be written to one at their uncompressed size.
+	usedBytes int64
+	// writeErr is the error of the last write of a report file to
+	// finish, nil if it succeeded or none has. While it is set, Append
+	// refuses reports, and the reports of that write wait in buf for
+	// the next flush.
+	writeErr error
+	// writeMu is held across each write of a report file, so writes
+	// never overlap: a failed write puts its reports back in buf before
+	// the next write takes them, which keeps them in the order they
+	// were appended.
+	writeMu sync.Mutex
+	// writers are the goroutines that write report files: flushLoop,
+	// and each one Append starts at flushSizeThreshold. OnStop waits
+	// for them before the final flush.
+	writers sync.WaitGroup
+}
+
+// New creates a Buffer and registers lifecycle hooks to
+// manage the data directory and flush goroutine.
+func New(
+	lc fx.Lifecycle,
+	params Params,
+) (*Buffer, error) {
+	dir := params.Config.DataDir
+	if dir == "" {
+		dir = defaultDataDir
+	}
+
+	b := &Buffer{
+		dataDir:     dir,
+		done:        make(chan struct{}),
+		fileCreated: func(*os.File) {},
+		log:         params.Logger.Get(),
+		maxBytes:    params.Config.DataDirMaxBytes,
+		now:         time.Now,
+	}
+
+	lc.Append(fx.Hook{
+		OnStart: func(_ context.Context) error {
+			// The errors name DATA_DIR, as config.New's do for the
+			// settings it checks.
+			err := os.MkdirAll(b.dataDir, dirPerms)
+			if err != nil {
+				return fmt.Errorf("DATA_DIR %q: %w", b.dataDir, err)
+			}
+
+			// Creating and removing a file stops the start when no
+			// file can be created in DATA_DIR, instead of the first
+			// write of a report file failing after reports were
+			// answered 200. A full disk is not caught: the empty
+			// file is still created.
+			f, err := os.CreateTemp(b.dataDir, "")
+			if err != nil {
+				return fmt.Errorf("DATA_DIR %q: %w", b.dataDir, err)
+			}
+
+			err = errors.Join(f.Close(), os.Remove(f.Name()))
+			if err != nil {
+				return fmt.Errorf("DATA_DIR %q: %w", b.dataDir, err)
+			}
+
+			// Report files left by earlier runs count too, and are
+			// the first to be deleted to make room.
+			files, err := reportFiles(b.dataDir)
+			if err != nil {
+				return fmt.Errorf("DATA_DIR %q: %w", b.dataDir, err)
+			}
+
+			b.mu.Lock()
+
+			b.files = files
+			for _, f := range files {
+				b.filesBytes += f.size
+			}
+
+			b.usedBytes = b.filesBytes
+
+			// The files may be past the cap, if it was lowered since
+			// the last run.
+			b.deleteOldestFiles(0)
+
+			b.mu.Unlock()
+
+			b.writers.Go(b.flushLoop)
+
+			return nil
+		},
+		OnStop: func(_ context.Context) error {
+			// stopOnce makes OnStop idempotent: a second
+			// invocation must not close an already-closed channel
+			// (which would panic) or flush again.
+			var err error
+
+			b.stopOnce.Do(func() {
+				// Closing done ends flushLoop after any flush it is
+				// running. Wait for it and for the writes Append
+				// started, which the process exiting would cut off;
+				// one that fails puts its reports back in buf, for the
+				// final flush to write.
+				close(b.done)
+				b.writers.Wait()
+
+				err = b.flushLocked()
+			})
+
+			// A failed final flush fails the stop, so the process
+			// exits non-zero.
+			return err
+		},
+	})
+
+	return b, nil
+}
+
+// Append marshals v as a single JSON line and appends it to
+// the buffer. It stores nothing and returns an error if the last
+// write of a report file failed, until a write succeeds again. If the
+// line would take usedBytes past maxBytes, the oldest report files are
+// deleted to make room; it stores nothing and returns ErrFull if that
+// cannot make room. If the buffer reaches the size threshold, it starts
+// a write of the reports in the background (see flushAtThreshold),
+// unless one it started is still waiting.
+func (b *Buffer) Append(v any) error {
+	line, err := json.Marshal(v)
+	if err != nil {
+		return fmt.Errorf("marshal report: %w", err)
+	}
+
+	lineBytes := int64(len(line)) + 1 // with its newline
+
+	b.mu.Lock()
+
+	if b.writeErr != nil {
+		err = fmt.Errorf("report files cannot be written: %w", b.writeErr)
+		b.mu.Unlock()
+
+		return err
+	}
+
+	b.deleteOldestFiles(lineBytes)
+
+	if b.usedBytes+lineBytes > b.maxBytes {
+		b.mu.Unlock()
+
+		return ErrFull
+	}
+
+	b.usedBytes += lineBytes
+	b.buf.Write(line)
+	b.buf.WriteByte('\n')
+
+	if b.buf.Len() >= flushSizeThreshold &&
+		b.thresholdWriteWaiting.CompareAndSwap(false, true) {
+		b.writers.Go(b.flushAtThreshold)
+	}
+
+	b.mu.Unlock()
+
+	return nil
+}
+
+// LastWriteFailed reports whether the last write of a report file to
+// finish failed: false at start until a write fails, and false again
+// once a write succeeds. While it is true, Append refuses reports.
+func (b *Buffer) LastWriteFailed() bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	return b.writeErr != nil
+}
+
+// deleteOldestFiles deletes report files, oldest first, until n more
+// bytes fit under maxBytes. It deletes none when the reports waiting
+// to be written leave no room for n even with every file gone, since
+// that would lose the files for nothing. The caller must hold b.mu.
+func (b *Buffer) deleteOldestFiles(n int64) {
+	for b.usedBytes+n > b.maxBytes && len(b.files) > 0 {
+		if b.usedBytes-b.filesBytes+n > b.maxBytes {
+			return
+		}
+
+		f := b.files[0]
+		b.files = b.files[1:]
+		b.filesBytes -= f.size
+
+		// A file already gone, deleted by hand, has freed its room too.
+		err := os.Remove(filepath.Join(b.dataDir, f.name))
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			// The file is still there, so it still counts. It is
+			// not tried again until the next start.
+			b.log.Error("delete report file failed",
+				"file", f.name, "error", err)
+
+			continue
+		}
+
+		b.usedBytes -= f.size
+		b.log.Info("deleted report file to make room",
+			"file", f.name, "bytes", f.size)
+	}
+}
+
+// flushLoop runs a ticker that periodically flushes buffered
+// data to disk until the done channel is closed.
+func (b *Buffer) flushLoop() {
+	ticker := time.NewTicker(flushInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			err := b.flushLocked()
+			if err != nil {
+				b.log.Error("flush reports failed", "error", err)
+			}
+		case <-b.done:
+			return
+		}
+	}
+}
+
+// flushLocked waits for any write of a report file under way, then
+// writes the reports waiting in the buffer to a new one.
+func (b *Buffer) flushLocked() error {
+	b.writeMu.Lock()
+	defer b.writeMu.Unlock()
+
+	return b.writeFile()
+}
+
+// flushAtThreshold is the write Append starts when the buffer reaches
+// flushSizeThreshold. Once any write under way is done, it lets Append
+// start another and writes the reports waiting in the buffer.
+func (b *Buffer) flushAtThreshold() {
+	b.writeMu.Lock()
+	defer b.writeMu.Unlock()
+
+	b.thresholdWriteWaiting.Store(false)
+
+	err := b.writeFile()
+	if err != nil {
+		b.log.Error("flush reports failed", "error", err)
+	}
+}
+
+// drainBuf copies the buffer contents and resets it.
+// The caller must hold b.mu.
+func (b *Buffer) drainBuf() []byte {
+	data := make([]byte, b.buf.Len())
+	copy(data, b.buf.Bytes())
+	b.buf.Reset()
+
+	return data
+}
+
+// writeFile takes the reports waiting in the buffer, if any, and writes
+// them to a new timestamped zstd-compressed JSONL file in the data
+// directory. If the write fails, it puts the reports back in the
+// buffer. The caller must hold b.writeMu.
+func (b *Buffer) writeFile() error {
+	b.mu.Lock()
+
+	if b.buf.Len() == 0 {
+		b.mu.Unlock()
+
+		return nil
+	}
+
+	data := b.drainBuf()
+	b.mu.Unlock()
+
+	// The timestamp comes first, so the names sort by time; the number
+	// after it tells apart files named in the same millisecond.
+	ts := b.now().UTC().Format("2006-01-02T15-04-05.000Z")
+	name := fmt.Sprintf("%s%s-%d%s", filePrefix, ts, b.seq.Add(1), fileSuffix)
+
+	size, err := b.createFile(filepath.Join(b.dataDir, name), data)
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	b.writeErr = err
+
+	if err != nil {
+		// The reports go back in front of any appended since, for the
+		// next flush to write, and still count at their uncompressed
+		// size.
+		appended := bytes.Clone(b.buf.Bytes())
+		b.buf.Reset()
+		b.buf.Write(data)
+		b.buf.Write(appended)
+
+		return err
+	}
+
+	// The reports no longer wait to be written, so they stop counting
+	// at their uncompressed size and count as the file, which from here
+	// on may be deleted to make room.
+	b.usedBytes -= int64(len(data))
+	b.usedBytes += size
+	b.filesBytes += size
+	b.files = append(b.files, reportFile{name: name, size: size})
+
+	return nil
+}
+
+// createFile creates the file at path holding data compressed with
+// zstd, and returns its size. If the write fails once the file is
+// created, it removes the file, so that a failed write leaves nothing
+// behind to take room.
+func (b *Buffer) createFile(path string, data []byte) (int64, error) {
+	// path is built from the operator-supplied dataDir plus a
+	// generated timestamp and number, so it carries no external input.
+	f, err := os.OpenFile( //nolint:gosec // see comment above
+		path,
+		os.O_WRONLY|os.O_CREATE|os.O_EXCL,
+		filePerms,
+	)
+	if err != nil {
+		return 0, fmt.Errorf("create report file: %w", err)
+	}
+
+	b.fileCreated(f)
+
+	size, err := writeCompressed(f, data)
+	if err != nil {
+		_ = f.Close()
+
+		return 0, errors.Join(err, os.Remove(path))
+	}
+
+	err = f.Close()
+	if err != nil {
+		err = fmt.Errorf("close report file: %w", err)
+
+		return 0, errors.Join(err, os.Remove(path))
+	}
+
+	return size, nil
+}
+
+// writeCompressed writes data to f compressed with zstd, and returns
+// the size of f.
+func writeCompressed(f *os.File, data []byte) (int64, error) {
+	enc, err := zstd.NewWriter(f)
+	if err != nil {
+		return 0, fmt.Errorf("create zstd encoder: %w", err)
+	}
+
+	_, err = enc.Write(data)
+	if err != nil {
+		_ = enc.Close()
+
+		return 0, fmt.Errorf("write compressed data: %w", err)
+	}
+
+	err = enc.Close()
+	if err != nil {
+		return 0, fmt.Errorf("close zstd encoder: %w", err)
+	}
+
+	info, err := f.Stat()
+	if err != nil {
+		return 0, fmt.Errorf("stat report file: %w", err)
+	}
+
+	return info.Size(), nil
+}
+
+// reportFiles returns the report files in dir, oldest first:
+// os.ReadDir sorts them by name, and the names sort by time.
+func reportFiles(dir string) ([]reportFile, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("read data dir: %w", err)
+	}
+
+	files := make([]reportFile, 0, len(entries))
+
+	for _, entry := range entries {
+		name := entry.Name()
+		if !strings.HasPrefix(name, filePrefix) ||
+			!strings.HasSuffix(name, fileSuffix) {
+			continue
+		}
+
+		info, err := entry.Info()
+		if err != nil {
+			return nil, fmt.Errorf("stat report file: %w", err)
+		}
+
+		files = append(files, reportFile{name: name, size: info.Size()})
+	}
+
+	return files, nil
+}