Evict archive writers on deletion and sweep idle archives (closes #89) (#95)

Per-webhook archive writers are now evicted when the webhook or its last
database target is deleted, and a background sweeper prunes expired rows from
idle archives that no longer receive writes. Archive files themselves are never
deleted.

internal/handlers/handlers.go

@@ -51,6 +51,7 @@ type HandlersParams struct {
 	Healthcheck  *healthcheck.Healthcheck
 	Session      *session.Session
 	Notifier     delivery.Notifier
+	Evictor      delivery.WebhookEvictor
 }
 
 // Handlers provides HTTP handler methods for all application
@@ -63,6 +64,7 @@ type Handlers struct {
 	dbMgr     *database.WebhookDBManager
 	session   *session.Session
 	notifier  delivery.Notifier
+	evictor   delivery.WebhookEvictor
 	templates map[string]*template.Template
 }
 
@@ -97,6 +99,7 @@ func New(
 	s.dbMgr = params.WebhookDBMgr
 	s.session = params.Session
 	s.notifier = params.Notifier
+	s.evictor = params.Evictor
 
 	// Parse all page templates once at startup
 	s.templates = map[string]*template.Template{

internal/handlers/handlers_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -24,6 +25,32 @@ type noopNotifier struct{}
 
 func (n *noopNotifier) Notify([]delivery.Task) {}
 
+// recordingEvictor is a delivery.WebhookEvictor that records
+// the webhook ids it was asked to evict, so a test can prove
+// that a deletion path reached the delivery engine.
+type recordingEvictor struct {
+	mu      sync.Mutex
+	evicted []string
+}
+
+func (r *recordingEvictor) EvictWebhook(webhookID string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	r.evicted = append(r.evicted, webhookID)
+}
+
+// Evicted returns a copy of the recorded webhook ids.
+func (r *recordingEvictor) Evicted() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	out := make([]string, len(r.evicted))
+	copy(out, r.evicted)
+
+	return out
+}
+
 func newTestApp(
 	t *testing.T,
 	targets ...any,
@@ -47,6 +74,12 @@ func newTestApp(
 			func() delivery.Notifier {
 				return &noopNotifier{}
 			},
+			func() *recordingEvictor {
+				return &recordingEvictor{}
+			},
+			func(r *recordingEvictor) delivery.WebhookEvictor {
+				return r
+			},
 			handlers.New,
 		),
 		fx.Populate(targets...),

internal/handlers/source_delete_test.go

@@ -0,0 +1,356 @@
+package handlers_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/go-chi/chi"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm/clause"
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/handlers"
+	"sneak.berlin/go/webhooker/internal/session"
+)
+
+const (
+	deleteTestUserID   = "test-user-id"
+	deleteTestUsername = "testuser"
+
+	// paramSourceID and paramTargetID are the chi URL parameter
+	// names the deletion handlers read.
+	paramSourceID = "sourceID"
+	paramTargetID = "targetID"
+)
+
+// seedWebhook inserts a webhook owned by the test user and
+// returns it.
+func seedWebhook(
+	t *testing.T,
+	db *database.Database,
+) *database.Webhook {
+	t.Helper()
+
+	wh := &database.Webhook{
+		UserID: deleteTestUserID,
+		Name:   "delete-me",
+	}
+
+	require.NoError(
+		t,
+		db.DB().Omit(clause.Associations).Create(wh).Error,
+	)
+
+	return wh
+}
+
+// seedTarget inserts a target of the given type for a webhook
+// and returns it.
+func seedTarget(
+	t *testing.T,
+	db *database.Database,
+	webhookID string,
+	targetType database.TargetType,
+) *database.Target {
+	t.Helper()
+
+	tgt := &database.Target{
+		WebhookID: webhookID,
+		Name:      "t-" + string(targetType),
+		Type:      targetType,
+		Active:    true,
+	}
+
+	require.NoError(
+		t,
+		db.DB().Omit(clause.Associations).Create(tgt).Error,
+	)
+
+	return tgt
+}
+
+// archivePathFor returns the archive database path the
+// delivery engine would use for a webhook: beside the webhook's
+// event database in the data directory.
+func archivePathFor(
+	t *testing.T,
+	mgr *database.WebhookDBManager,
+	webhookID string,
+) string {
+	t.Helper()
+
+	return filepath.Join(
+		filepath.Dir(mgr.DBPath(webhookID)),
+		"archive-"+webhookID+".db",
+	)
+}
+
+// writeArchivePlaceholder creates a stand-in archive file so a
+// test can assert the file survives webhook deletion.
+func writeArchivePlaceholder(path string) error {
+	return os.WriteFile(path, []byte("archive"), 0o600)
+}
+
+// postRequest builds an authenticated POST request carrying the
+// given chi URL parameters.
+func postRequest(
+	path string,
+	cookies []*http.Cookie,
+	params map[string]string,
+) *http.Request {
+	req := httptest.NewRequestWithContext(
+		context.Background(), http.MethodPost, path, nil,
+	)
+
+	for _, c := range cookies {
+		req.AddCookie(c)
+	}
+
+	rctx := chi.NewRouteContext()
+	for k, v := range params {
+		rctx.URLParams.Add(k, v)
+	}
+
+	return req.WithContext(
+		context.WithValue(req.Context(), chi.RouteCtxKey, rctx),
+	)
+}
+
+// TestHandleSourceDelete_EvictsArchiveWriter proves that
+// deleting a webhook reaches the delivery engine and releases
+// the webhook's archive writer, exercised through the real
+// deletion handler rather than by calling the evictor directly.
+func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		ev   *recordingEvictor
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &ev)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := seedWebhook(t, db)
+	seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/source/"+wh.ID+"/delete",
+		cookies,
+		map[string]string{paramSourceID: wh.ID},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleSourceDelete().ServeHTTP(w, req)
+
+	require.Equal(t, http.StatusSeeOther, w.Code)
+	assert.Equal(
+		t, []string{wh.ID}, ev.Evicted(),
+		"deleting a webhook should evict its archive writer",
+	)
+}
+
+// TestHandleSourceDelete_KeepsArchiveFile proves that deleting
+// a webhook does not remove its archive database file: the
+// archive is long-term storage the operator owns.
+func TestHandleSourceDelete_KeepsArchiveFile(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		mgr  *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &mgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := seedWebhook(t, db)
+
+	// Place an archive file where the delivery engine would.
+	archivePath := archivePathFor(t, mgr, wh.ID)
+	require.NoError(
+		t,
+		writeArchivePlaceholder(archivePath),
+	)
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/source/"+wh.ID+"/delete",
+		cookies,
+		map[string]string{paramSourceID: wh.ID},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleSourceDelete().ServeHTTP(w, req)
+
+	require.Equal(t, http.StatusSeeOther, w.Code)
+	assert.FileExists(
+		t, archivePath,
+		"webhook deletion must not destroy the archive file",
+	)
+}
+
+// TestHandleTargetDelete_EvictsWhenLastDatabaseTargetGone
+// proves that removing the last database target releases the
+// archive writer.
+func TestHandleTargetDelete_EvictsWhenLastDatabaseTargetGone(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		ev   *recordingEvictor
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &ev)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := seedWebhook(t, db)
+	tgt := seedTarget(
+		t, db, wh.ID, database.TargetTypeDatabase,
+	)
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/source/"+wh.ID+"/targets/"+tgt.ID+"/delete",
+		cookies,
+		map[string]string{
+			paramSourceID: wh.ID,
+			paramTargetID: tgt.ID,
+		},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleTargetDelete().ServeHTTP(w, req)
+
+	require.Equal(t, http.StatusSeeOther, w.Code)
+	assert.Equal(
+		t, []string{wh.ID}, ev.Evicted(),
+		"removing the last database target should evict",
+	)
+}
+
+// TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains
+// proves that deleting one of several database targets leaves
+// the still-needed archive writer alone: the surviving target
+// keeps archiving to the same file, so the writer must stay.
+func TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		ev   *recordingEvictor
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &ev)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := seedWebhook(t, db)
+	doomed := seedTarget(
+		t, db, wh.ID, database.TargetTypeDatabase,
+	)
+	seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/source/"+wh.ID+"/targets/"+doomed.ID+"/delete",
+		cookies,
+		map[string]string{
+			paramSourceID: wh.ID,
+			paramTargetID: doomed.ID,
+		},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleTargetDelete().ServeHTTP(w, req)
+
+	require.Equal(t, http.StatusSeeOther, w.Code)
+	assert.Empty(
+		t, ev.Evicted(),
+		"a second database target still needs the writer",
+	)
+}
+
+// TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted proves
+// that deleting a target of an unrelated type leaves a
+// still-needed archive writer alone: the webhook's database
+// target is untouched, so its writer must stay.
+func TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		ev   *recordingEvictor
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &ev)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := seedWebhook(t, db)
+	seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
+	other := seedTarget(t, db, wh.ID, database.TargetTypeLog)
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/source/"+wh.ID+"/targets/"+other.ID+"/delete",
+		cookies,
+		map[string]string{
+			paramSourceID: wh.ID,
+			paramTargetID: other.ID,
+		},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleTargetDelete().ServeHTTP(w, req)
+
+	require.Equal(t, http.StatusSeeOther, w.Code)
+	assert.Empty(
+		t, ev.Evicted(),
+		"a surviving database target must keep its writer",
+	)
+}

internal/handlers/source_management.go

@@ -533,6 +533,13 @@ func (h *Handlers) deleteWebhookResources(
 		return
 	}
 
+	// Release the delivery engine's per-webhook archiving state
+	// so a deleted webhook's archive writer (and any handle open
+	// within its debounce window) does not linger for the
+	// process lifetime. The archive file itself is deliberately
+	// left on disk; see evictArchiveWriter.
+	h.evictArchiveWriter(webhook.ID)
+
 	err = h.dbMgr.DeleteDB(webhook.ID)
 	if err != nil {
 		h.log.Error(
@@ -551,6 +558,64 @@ func (h *Handlers) deleteWebhookResources(
 	http.Redirect(w, r, "/sources", http.StatusSeeOther)
 }
 
+// evictArchiveWriter asks the delivery engine to drop its
+// cached archive writer for a webhook, closing the archive file
+// handle.
+//
+// The archive database file is NOT deleted. Unlike the event
+// database — which is per-webhook working storage and is
+// hard-deleted with the webhook — an archive is explicitly
+// long-term storage that an operator may want to keep or move
+// away for offline retention. Destroying it as a side effect of
+// deleting a webhook would be a surprising and unrecoverable
+// data loss, so the file is left for the operator to handle.
+func (h *Handlers) evictArchiveWriter(webhookID string) {
+	if h.evictor == nil {
+		return
+	}
+
+	h.evictor.EvictWebhook(webhookID)
+}
+
+// evictArchiveWriterIfUnused releases a webhook's archive
+// writer once the webhook has no database target left to feed
+// it.
+//
+// It is called after any child resource of a webhook is
+// deleted, and is correct without knowing which kind was: it
+// evicts only when no database target remains, so deleting one
+// of several database targets — or deleting an unrelated
+// target type — leaves a still-needed writer alone. When no
+// database target ever existed there is no writer and eviction
+// is a no-op. Soft-deleted targets are excluded by GORM's
+// default scope, so the row just deleted is not counted.
+func (h *Handlers) evictArchiveWriterIfUnused(webhookID string) {
+	var remaining int64
+
+	err := h.db.DB().
+		Model(&database.Target{}).
+		Where(
+			"webhook_id = ? AND type = ?",
+			webhookID, database.TargetTypeDatabase,
+		).
+		Count(&remaining).Error
+	if err != nil {
+		h.log.Error(
+			"failed to count remaining database targets",
+			"webhook_id", webhookID,
+			"error", err,
+		)
+
+		return
+	}
+
+	if remaining > 0 {
+		return
+	}
+
+	h.evictArchiveWriter(webhookID)
+}
+
 // HandleSourceLogs shows the request/response logs for a
 // webhook.
 func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
@@ -1024,23 +1089,31 @@ func (h *Handlers) HandleEntrypointDelete() http.HandlerFunc {
 	return h.deleteChildResource(
 		"entrypointID", &database.Entrypoint{},
 		"failed to delete entrypoint",
+		nil,
 	)
 }
 
-// HandleTargetDelete handles deleting a target.
+// HandleTargetDelete handles deleting a target. Deleting the
+// last database target of a webhook leaves its archive writer
+// with nothing to write, so the writer is evicted and its
+// handle closed; the archive file is left on disk.
 func (h *Handlers) HandleTargetDelete() http.HandlerFunc {
 	return h.deleteChildResource(
 		"targetID", &database.Target{},
 		"failed to delete target",
+		h.evictArchiveWriterIfUnused,
 	)
 }
 
 // deleteChildResource returns a handler that deletes a child
-// resource (entrypoint or target) belonging to a webhook.
+// resource (entrypoint or target) belonging to a webhook. The
+// optional afterDelete hook runs with the webhook's id once the
+// delete has succeeded, before the redirect.
 func (h *Handlers) deleteChildResource(
 	idParam string,
 	model any,
 	errMsg string,
+	afterDelete func(webhookID string),
 ) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		userID, ok := h.getUserID(r)
@@ -1080,6 +1153,10 @@ func (h *Handlers) deleteChildResource(
 			return
 		}
 
+		if afterDelete != nil {
+			afterDelete(webhook.ID)
+		}
+
 		http.Redirect(
 			w, r,
 			"/source/"+webhook.ID,