Try every event database file on delete and say which was left (closes #275)

DeleteDB stopped at the first file it could not remove, and the
webhook delete handler logged one message for every failure. Once
the database file itself is gone, its events are lost, and telling
the operator only that a file was left behind suggests otherwise.

DeleteDB now tries the database file and both sidecars, and its
error wraps ErrEventDBNotRemoved when the database file is still on
disk, or ErrSidecarNotRemoved when only a -wal or -shm sidecar is
left. The handler logs a different message for each.

Model: opus-5-5

internal/handlers/source_delete_test.go

@@ -1,8 +1,10 @@
 package handlers_test
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -466,6 +468,66 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
 	)
 }
 
+// TestHandleSourceDelete_LeftoverSidecar proves that when the event
+// database file is removed but a sidecar beside it is not, the
+// operator is told the events are gone, never that the event
+// database file is still there.
+func TestHandleSourceDelete_LeftoverSidecar(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		mgr  *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &mgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	logs := new(bytes.Buffer)
+	h.SetLogForTest(slog.New(slog.NewTextHandler(logs, nil)))
+
+	wh := seedWebhook(t, db)
+
+	require.NoError(t, mgr.CreateDB(wh.ID))
+	// Closing removes the sidecars, so the -wal below is the only
+	// one there.
+	require.NoError(t, mgr.CloseAll())
+
+	// A non-empty directory in the -wal file's place, which
+	// os.Remove cannot remove whoever runs the test.
+	eventDBPath := mgr.DBPath(wh.ID)
+	require.NoError(t, os.MkdirAll(
+		filepath.Join(eventDBPath+"-wal", "keep"), 0o700,
+	))
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/hook/"+wh.ID+"/delete",
+		cookies,
+		map[string]string{paramSourceID: wh.ID},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleSourceDelete().ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusInternalServerError, w.Code)
+	assert.NoFileExists(t, eventDBPath)
+	assert.Contains(t, logs.String(), "its events are gone")
+	assert.Contains(t, logs.String(), eventDBPath+"-wal")
+	assert.NotContains(
+		t, logs.String(), "event database file is still on disk",
+		"the events are gone, so the operator must not be told "+
+			"the event database file survived",
+	)
+}
+
 // TestHandleTargetDelete_EvictsThatTarget proves that deleting a
 // database target releases that target's archive writer and no
 // other: the webhook's other database target keeps its own.

internal/handlers/source_management.go

@@ -762,13 +762,21 @@ func (h *Handlers) deleteWebhookResources(
 	err = h.dbMgr.DeleteDB(webhook.ID)
 	if err != nil {
 		// The configuration is committed, so the webhook is gone,
-		// but its event database file is still on disk with
+		// but a file of its event database is still on disk with
 		// nothing referencing it. Report the failure rather than
 		// redirecting as though everything succeeded: the file
 		// needs removing by hand, and the logged error names it.
-		h.serverError(
-			w, r, "failed to delete webhook event database", err,
-		)
+		// When only a sidecar is left, the events are already
+		// gone, and the message must not suggest they survive.
+		msg := "webhook deleted, but its event database file is " +
+			"still on disk; remove it by hand"
+		if errors.Is(err, database.ErrSidecarNotRemoved) {
+			msg = "webhook deleted and its events are gone, but a " +
+				"-wal or -shm sidecar of its event database is " +
+				"still on disk; remove it by hand"
+		}
+
+		h.serverError(w, r, msg, err)
 
 		return
 	}