webhooker/internal/delivery/target_database_evict_test.go

package delivery_test

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"sneak.berlin/go/webhooker/internal/database"
	"sneak.berlin/go/webhooker/internal/delivery"
)

// deliverTo archives one event to a database target, leaving the
// target's writer cached with its handle open.
func deliverTo(
	t *testing.T, env *archiveEnv, tgt *database.Target,
) {
	t.Helper()

	webhookDB := testWebhookDB(t)
	event := seedEvent(t, webhookDB, `{"archived":true}`)

	env.eng.ExportDeliverDatabase(
		webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
	)
}

// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting
// a webhook drops the archive writers of its database targets
// from the registry and closes their open handles, rather than
// leaving them alive for the process lifetime, and leaves another
// webhook's writer alone.
func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
	t.Parallel()

	env := setupArchiveTest(t)
	first := env.seedDatabaseTarget(t, "")
	second := env.addDatabaseTarget(t, first.WebhookID, "")
	other := env.seedDatabaseTarget(t, "")

	for _, tgt := range []*database.Target{first, second, other} {
		deliverTo(t, env, tgt)

		require.True(
			t, env.eng.ExportArchiveHandleOpen(tgt.ID),
			"the writer should hold an open handle after a write",
		)
	}

	env.eng.EvictWebhook(first.WebhookID)

	for _, tgt := range []*database.Target{first, second} {
		assert.False(
			t, env.eng.ExportHasArchiveWriter(tgt.ID),
			"eviction should remove the registry entry",
		)
		assert.False(
			t, env.eng.ExportArchiveHandleOpen(tgt.ID),
			"eviction should close the archive handle",
		)
		assert.FileExists(
			t, env.archivePath(tgt),
			"eviction must not delete the archive file",
		)
	}

	assert.True(
		t, env.eng.ExportArchiveHandleOpen(other.ID),
		"another webhook's writer must be left alone",
	)
}

// TestEvictTarget_LeavesOtherTargets proves that evicting one
// database target leaves the writer of another target of the same
// webhook in place.
func TestEvictTarget_LeavesOtherTargets(t *testing.T) {
	t.Parallel()

	env := setupArchiveTest(t)
	doomed := env.seedDatabaseTarget(t, "")
	kept := env.addDatabaseTarget(t, doomed.WebhookID, "")

	deliverTo(t, env, doomed)
	deliverTo(t, env, kept)

	env.eng.EvictTarget(doomed.ID)

	assert.False(t, env.eng.ExportHasArchiveWriter(doomed.ID))
	assert.FileExists(
		t, env.archivePath(doomed),
		"eviction must not delete the archive file",
	)
	assert.True(
		t, env.eng.ExportArchiveHandleOpen(kept.ID),
		"the other target's writer must be left alone",
	)
}

// TestEvictWebhook_UnknownWebhookIsNoOp proves eviction is safe
// for the common case of a webhook or target that never had an
// archive writer, and that repeating it does not panic.
func TestEvictWebhook_UnknownWebhookIsNoOp(t *testing.T) {
	t.Parallel()

	env := setupArchiveTest(t)

	assert.NotPanics(t, func() {
		env.eng.EvictWebhook("no-such-webhook")
		env.eng.EvictWebhook("no-such-webhook")
		env.eng.EvictTarget("no-such-target")
		env.eng.EvictTarget("no-such-target")
	})

	assert.False(
		t, env.eng.ExportHasArchiveWriter("no-such-target"),
		"eviction must not create a writer",
	)
}

// evictTestRow builds an archive row for the eviction tests.
func evictTestRow(eventID string) delivery.ExportArchivedEvent {
	return delivery.ExportArchivedEvent{
		EventID:   eventID,
		WebhookID: "wh-evict",
		Method:    http.MethodPost,
		Body:      `{"seeded":true}`,
	}
}

// TestEvictedWriter_WriteDoesNotReopenFile is the direct test of
// the evicted guard on the write path. A writer that has left
// the registry is held by nobody, so a handle it opened could
// never be closed again: it must refuse the write outright
// rather than recreate the archive behind the registry's back.
//
// The archive file is removed before the eviction, so an
// unguarded write is unmistakable — it recreates the file.
func TestEvictedWriter_WriteDoesNotReopenFile(t *testing.T) {
	t.Parallel()

	path := filepath.Join(t.TempDir(), "archive-evicted.db")

	w := delivery.NewExportArchiveWriter(
		path, archiveTestLogger(), 0,
	)

	require.NoError(t, w.Write(evictTestRow("ev-1"), 0))
	require.FileExists(t, path)

	// The operator moves the archive away for offline retention,
	// which the write path would ordinarily undo on the next
	// write by recreating the file.
	require.NoError(t, os.Remove(path))

	w.Evict()

	err := w.Write(evictTestRow("ev-2"), 0)

	require.ErrorIs(
		t, err, delivery.ErrExportArchiveWriterEvicted,
		"an evicted writer must refuse writes",
	)
	assert.NoFileExists(
		t, path,
		"an evicted writer must not reopen (or recreate) the "+
			"archive file",
	)
	assert.False(
		t, w.HandleOpen(),
		"an evicted writer must hold no handle",
	)
}

// TestEvictedWriter_SweepDoesNotReopenFile is the same test for
// the sweep path: an idle sweep that reaches a writer already
// evicted underneath it must return the sentinel rather than
// reopen a file nothing owns.
func TestEvictedWriter_SweepDoesNotReopenFile(t *testing.T) {
	t.Parallel()

	path := filepath.Join(t.TempDir(), "archive-evicted.db")

	w := delivery.NewExportArchiveWriter(
		path, archiveTestLogger(), 0,
	)

	require.NoError(t, w.Write(evictTestRow("ev-1"), 0))
	require.FileExists(t, path)

	w.Evict()

	err := w.SweepExpired(time.Hour)

	require.ErrorIs(
		t, err, delivery.ErrExportArchiveWriterEvicted,
		"an evicted writer must refuse an idle sweep",
	)
	assert.False(
		t, w.HandleOpen(),
		"a refused sweep must not leave a handle open",
	)
}

// racingWrites drives a pack of goroutines writing to one
// archive writer until each is refused, so an eviction on the
// test goroutine has to take the writer's mutex away from writes
// that are already contending for it.
type racingWrites struct {
	wg         sync.WaitGroup
	mu         sync.Mutex
	sawEvicted bool
	otherErr   error
	started    chan struct{}
}

// racingWriteGoroutines is how many goroutines contend for the
// writer's mutex while the eviction lands.
const racingWriteGoroutines = 4

// startRacingWrites launches the writing goroutines. Each writes
// in a loop and stops at its first error, recording whether that
// error was the eviction sentinel. The deadline is a backstop
// against a hang, not a timing assumption: the first write after
// the eviction is refused.
func startRacingWrites(
	w *delivery.ExportArchiveWriter,
) *racingWrites {
	r := &racingWrites{
		started: make(chan struct{}, racingWriteGoroutines),
	}

	deadline := time.Now().Add(10 * time.Second)

	r.wg.Add(racingWriteGoroutines)

	for i := range racingWriteGoroutines {
		go func() {
			defer r.wg.Done()

			first := true

			for time.Now().Before(deadline) {
				err := w.Write(
					evictTestRow(fmt.Sprintf("ev-%d", i)), 0,
				)

				if first {
					r.started <- struct{}{}

					first = false
				}

				if err == nil {
					continue
				}

				r.record(err)

				return
			}
		}()
	}

	return r
}

// record classifies the error that stopped one goroutine.
func (r *racingWrites) record(err error) {
	r.mu.Lock()
	defer r.mu.Unlock()

	if errors.Is(err, delivery.ErrExportArchiveWriterEvicted) {
		r.sawEvicted = true

		return
	}

	r.otherErr = err
}

// awaitFirstWrite blocks until at least one write has run, so
// the eviction that follows is a genuine race.
func (r *racingWrites) awaitFirstWrite() {
	<-r.started
}

// wait joins the goroutines and reports whether any write was
// refused with the eviction sentinel, plus any unexpected error.
func (r *racingWrites) wait() (bool, error) {
	r.wg.Wait()

	r.mu.Lock()
	defer r.mu.Unlock()

	return r.sawEvicted, r.otherErr
}

// TestEvictWebhook_RacingWriteDoesNotReopenHandle exercises the
// interleaving the evicted flag exists for: writes already
// contending for the writer's mutex when the eviction takes it.
// The write that wins the mutex after the eviction must abandon
// its work rather than reopen the archive, leaving the writer
// permanently handle-free. Run under -race.
func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
	t *testing.T,
) {
	t.Parallel()

	env := setupArchiveTest(t)
	tgt := env.seedDatabaseTarget(t, "")

	// Prime the registry so the test can hold the very writer the
	// eviction is about to detach.
	deliverTo(t, env, tgt)

	w := env.eng.ExportArchiveWriterFor(tgt.ID)
	require.NotNil(t, w)
	require.True(t, w.HandleOpen())

	race := startRacingWrites(w)

	// Evict only once writes are genuinely in flight, so the
	// eviction has to contend for the writer's mutex.
	race.awaitFirstWrite()

	env.eng.EvictWebhook(tgt.WebhookID)

	sawEvicted, otherErr := race.wait()

	require.NoError(t, otherErr)
	assert.True(
		t, sawEvicted,
		"a write after eviction must be refused",
	)
	assert.False(
		t, w.HandleOpen(),
		"no write may reopen the archive once the writer has "+
			"been evicted",
	)
	assert.False(
		t, env.eng.ExportHasArchiveWriter(tgt.ID),
		"the registry entry must stay gone",
	)
}

// TestEvictWebhook_LaterDeliveryRecreatesWriter proves eviction
// does not break archiving for a target that is still alive: a
// subsequent delivery gets a brand new writer from the registry.
// It says nothing about the evicted writer itself — that is what
// TestEvictedWriter_WriteDoesNotReopenFile covers.
func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
	t.Parallel()

	env := setupArchiveTest(t)
	tgt := env.seedDatabaseTarget(t, "")

	deliverTo(t, env, tgt)
	require.True(t, env.eng.ExportHasArchiveWriter(tgt.ID))

	env.eng.EvictWebhook(tgt.WebhookID)

	// A fresh delivery for the same target gets a brand new
	// writer from the registry, so archiving keeps working.
	deliverTo(t, env, tgt)

	assert.True(
		t, env.eng.ExportHasArchiveWriter(tgt.ID),
		"a later delivery should recreate the writer",
	)
}

// TestEngineStop_WriteAfterStopIsRefused proves the engine's stop
// closes each archive writer the way deleting its webhook does: a
// write that reaches a writer after the stop is refused, reopens
// nothing and adds no row.
func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
	t.Parallel()

	env := setupArchiveTest(t)
	tgt := env.seedDatabaseTarget(t, "")

	deliverTo(t, env, tgt)

	w := env.eng.ExportArchiveWriterFor(tgt.ID)
	require.NotNil(t, w)
	require.True(t, w.HandleOpen())

	require.NoError(t, env.eng.ExportStop(context.Background()))

	err := w.Write(evictTestRow("ev-after-stop"), 0)

	require.ErrorIs(
		t, err, delivery.ErrExportArchiveWriterEvicted,
		"a write after the stop must be refused",
	)
	assert.False(
		t, w.HandleOpen(),
		"a refused write must not reopen the archive",
	)
	assert.False(
		t, env.eng.ExportHasArchiveWriter(tgt.ID),
		"the stop should empty the registry",
	)

	count, err := countArchivedRows(w.Path())
	require.NoError(t, err)
	assert.Equal(
		t, int64(1), count, "the refused row must not be written",
	)
}