Try every event database file on delete and say which was left (closes #275)

DeleteDB stopped at the first file it could not remove, and the
webhook delete handler logged one message for every failure. Once
the database file itself is gone, its events are lost, and telling
the operator only that a file was left behind suggests otherwise.

DeleteDB now tries the database file and both sidecars, and its
error wraps ErrEventDBNotRemoved when the database file is still on
disk, or ErrSidecarNotRemoved when only a -wal or -shm sidecar is
left. The handler logs a different message for each.

Model: opus-5-5

internal/handlers/source_delete_test.go

@@ -1,8 +1,10 @@
 package handlers_test
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -466,6 +468,121 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
 	)
 }
 
+// TestHandleSourceDelete_LeftoverSidecar proves that when the event
+// database file is removed but a sidecar beside it is not, the
+// operator is told the events are gone, never that the event
+// database file is still there.
+func TestHandleSourceDelete_LeftoverSidecar(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		mgr  *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &mgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	logs := new(bytes.Buffer)
+	h.SetLogForTest(slog.New(slog.NewTextHandler(logs, nil)))
+
+	wh := seedWebhook(t, db)
+
+	require.NoError(t, mgr.CreateDB(wh.ID))
+	// Closing removes the sidecars, so the -wal below is the only
+	// one there.
+	require.NoError(t, mgr.CloseAll())
+
+	// A non-empty directory in the -wal file's place, which
+	// os.Remove cannot remove whoever runs the test.
+	eventDBPath := mgr.DBPath(wh.ID)
+	require.NoError(t, os.MkdirAll(
+		filepath.Join(eventDBPath+"-wal", "keep"), 0o700,
+	))
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/hook/"+wh.ID+"/delete",
+		cookies,
+		map[string]string{paramSourceID: wh.ID},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleSourceDelete().ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusInternalServerError, w.Code)
+	assert.NoFileExists(t, eventDBPath)
+	assert.Contains(t, logs.String(), "its events are gone")
+	assert.Contains(t, logs.String(), eventDBPath+"-wal")
+	assert.NotContains(
+		t, logs.String(), handlers.EventDBLeftMsgForTest,
+		"the events are gone, so the operator must not be told "+
+			"the event database file survived",
+	)
+}
+
+// TestHandleSourceDelete_LeftoverDatabaseFile proves that when the
+// event database file itself cannot be removed, the operator is told
+// it is still on disk, never that its events are gone.
+func TestHandleSourceDelete_LeftoverDatabaseFile(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h    *handlers.Handlers
+		sess *session.Session
+		db   *database.Database
+		mgr  *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &mgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	logs := new(bytes.Buffer)
+	h.SetLogForTest(slog.New(slog.NewTextHandler(logs, nil)))
+
+	wh := seedWebhook(t, db)
+
+	// A non-empty directory in the database file's place, which
+	// os.Remove cannot remove whoever runs the test.
+	eventDBPath := mgr.DBPath(wh.ID)
+	require.NoError(t, os.MkdirAll(
+		filepath.Join(eventDBPath, "keep"), 0o700,
+	))
+
+	cookies := authenticatedCookies(
+		t, sess, deleteTestUserID, deleteTestUsername,
+	)
+
+	req := postRequest(
+		"/hook/"+wh.ID+"/delete",
+		cookies,
+		map[string]string{paramSourceID: wh.ID},
+	)
+	w := httptest.NewRecorder()
+
+	h.HandleSourceDelete().ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusInternalServerError, w.Code)
+	assert.Contains(
+		t, logs.String(), "event database file is still on disk",
+	)
+	assert.Contains(t, logs.String(), eventDBPath)
+	assert.NotContains(
+		t, logs.String(), handlers.SidecarLeftMsgForTest,
+		"the database file is still on disk, so the operator must "+
+			"not be told its events are gone",
+	)
+}
+
 // TestHandleTargetDelete_EvictsThatTarget proves that deleting a
 // database target releases that target's archive writer and no
 // other: the webhook's other database target keeps its own.