Try every event database file on delete and say which was left (closes #275)

DeleteDB returned on the first event database file it failed to remove, so with WAL sidecars present a failure could be reported as "the file was left behind" after the events themselves were already gone. It now tries the database file and both sidecars, and its error says which case happened and names each file left: the database file is still on disk, or it is gone (the events are lost) and only a -wal or -shm sidecar remains. The webhook delete handler logs a different message for each and never says the events survive when they do not. Tests cover a normal delete with both sidecars, and a failed removal of the database file or a sidecar, through DeleteDB and the handler.

Model: opus-5-5

internal/handlers/export_test.go

@@ -36,6 +36,15 @@ const MaxRenderedAttemptsForTest = maxRenderedAttempts
 // the handlers enforce rather than a number copied beside it.
 const MaxTargetRetriesForTest = maxTargetRetries
 
+// EventDBLeftMsgForTest and SidecarLeftMsgForTest expose the two
+// messages the webhook delete handler logs when a file of the event
+// database is left on disk, so a test checking that one is absent
+// checks for the handler's own wording.
+const (
+	EventDBLeftMsgForTest = eventDBLeftMsg
+	SidecarLeftMsgForTest = sidecarLeftMsg
+)
+
 // PageOrFirstForTest exposes pageOrFirst for use in the handlers_test
 // package.
 func PageOrFirstForTest(s string) int {

internal/handlers/source_delete_test.go

@@ -1,8 +1,10 @@
 package handlers_test
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -466,6 +468,121 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
 	)
 }
 
+// TestHandleSourceDelete_LeftoverSidecar proves that when the event
+// database file is removed but a sidecar beside it is not, the
+// operator is told the events are gone, never that the event
+// database file is still there.
+func TestHandleSourceDelete_LeftoverSidecar(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		mgr  *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &mgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	logs := new(bytes.Buffer)
+	h.SetLogForTest(slog.New(slog.NewTextHandler(logs, nil)))
+
+	wh := seedWebhook(t, db)
+
+	require.NoError(t, mgr.CreateDB(wh.ID))
+	// Closing removes the sidecars, so the -wal below is the only
+	// one there.
+	require.NoError(t, mgr.CloseAll())
+
+	// A non-empty directory in the -wal file's place, which
+	// os.Remove cannot remove whoever runs the test.
+	eventDBPath := mgr.DBPath(wh.ID)
+	require.NoError(t, os.MkdirAll(
+		filepath.Join(eventDBPath+"-wal", "keep"), 0o700,
+	))
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/hook/"+wh.ID+"/delete",
+		cookies,
+		map[string]string{paramSourceID: wh.ID},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleSourceDelete().ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusInternalServerError, w.Code)
+	assert.NoFileExists(t, eventDBPath)
+	assert.Contains(t, logs.String(), "its events are gone")
+	assert.Contains(t, logs.String(), eventDBPath+"-wal")
+	assert.NotContains(
+		t, logs.String(), handlers.EventDBLeftMsgForTest,
+		"the events are gone, so the operator must not be told "+
+			"the event database file survived",
+	)
+}
+
+// TestHandleSourceDelete_LeftoverDatabaseFile proves that when the
+// event database file itself cannot be removed, the operator is told
+// it is still on disk, never that its events are gone.
+func TestHandleSourceDelete_LeftoverDatabaseFile(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		mgr  *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &mgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	logs := new(bytes.Buffer)
+	h.SetLogForTest(slog.New(slog.NewTextHandler(logs, nil)))
+
+	wh := seedWebhook(t, db)
+
+	// A non-empty directory in the database file's place, which
+	// os.Remove cannot remove whoever runs the test.
+	eventDBPath := mgr.DBPath(wh.ID)
+	require.NoError(t, os.MkdirAll(
+		filepath.Join(eventDBPath, "keep"), 0o700,
+	))
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/hook/"+wh.ID+"/delete",
+		cookies,
+		map[string]string{paramSourceID: wh.ID},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleSourceDelete().ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusInternalServerError, w.Code)
+	assert.Contains(
+		t, logs.String(), "event database file is still on disk",
+	)
+	assert.Contains(t, logs.String(), eventDBPath)
+	assert.NotContains(
+		t, logs.String(), handlers.SidecarLeftMsgForTest,
+		"the database file is still on disk, so the operator must "+
+			"not be told its events are gone",
+	)
+}
+
 // TestHandleTargetDelete_EvictsThatTarget proves that deleting a
 // database target releases that target's archive writer and no
 // other: the webhook's other database target keeps its own.

internal/handlers/source_management.go

@@ -723,6 +723,17 @@ func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
 	}
 }
 
+// The messages deleteWebhookResources logs when a file of the event
+// database cannot be removed: the database file itself, or only a
+// sidecar once the database file is gone.
+const (
+	eventDBLeftMsg = "webhook deleted, but its event database file is " +
+		"still on disk; remove it by hand"
+	sidecarLeftMsg = "webhook deleted and its events are gone, but a " +
+		"-wal or -shm sidecar of its event database is " +
+		"still on disk; remove it by hand"
+)
+
 // deleteWebhookResources soft-deletes config and hard-deletes
 // the per-webhook event database.
 func (h *Handlers) deleteWebhookResources(
@@ -762,13 +773,18 @@ func (h *Handlers) deleteWebhookResources(
 	err = h.dbMgr.DeleteDB(webhook.ID)
 	if err != nil {
 		// The configuration is committed, so the webhook is gone,
-		// but its event database file is still on disk with
+		// but a file of its event database is still on disk with
 		// nothing referencing it. Report the failure rather than
 		// redirecting as though everything succeeded: the file
 		// needs removing by hand, and the logged error names it.
-		h.serverError(
-			w, r, "failed to delete webhook event database", err,
-		)
+		// When only a sidecar is left, the events are already
+		// gone, and the message must not suggest they survive.
+		msg := eventDBLeftMsg
+		if errors.Is(err, database.ErrSidecarNotRemoved) {
+			msg = sidecarLeftMsg
+		}
+
+		h.serverError(w, r, msg, err)
 
 		return
 	}