Name each database target's archive for its webhook and target (closes #376)

Each database target now writes its own archive file, archive-WEBHOOKNAME-TARGETNAME-TARGETID.db, named by delivery.ArchiveFileName, in place of one archive per webhook keyed on its UUID. Renaming a webhook or a target renames its archive files (with any -wal and -shm) before the new name is saved, never over an existing file, and moves every one back if a rename or the save fails. The webhook edit, the target edit and target creation share one lock so no two interleave. Deleting a webhook or target leaves its files on disk. Nothing looks for the old archive-WEBHOOKID.db files. The README gives the naming and the recovery steps.

Model: opus-5-5

internal/handlers/source_management_test.go

@@ -2,16 +2,20 @@ package handlers_test
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/go-chi/chi"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
 	"sneak.berlin/go/webhooker/internal/database"
 	"sneak.berlin/go/webhooker/internal/handlers"
@@ -187,6 +191,7 @@ func storedRetentionDays(
 type sourceTestEnv struct {
 	handlers *handlers.Handlers
 	db       *database.Database
+	archives *recordingArchives
 	cookies  []*http.Cookie
 }
 
@@ -199,7 +204,9 @@ func setupSourceTest(t *testing.T) *sourceTestEnv {
 
 	var db *database.Database
 
-	app := newTestApp(t, &h, &sess, &db)
+	var archives *recordingArchives
+
+	app := newTestApp(t, &h, &sess, &db, &archives)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -207,6 +214,7 @@ func setupSourceTest(t *testing.T) *sourceTestEnv {
 	return &sourceTestEnv{
 		handlers: h,
 		db:       db,
+		archives: archives,
 		cookies: authenticatedCookies(
 			t, sess, sourceTestUserID, "sourceuser",
 		),
@@ -498,6 +506,301 @@ func TestHandleSourceEditSubmit_EmptyRetentionLeavesValueUnchanged(
 	assert.Equal(t, 7, storedRetentionDays(t, env.db, wh.ID))
 }
 
+// renamedWebhookName is the name the rename tests give a webhook.
+const renamedWebhookName = "Renamed"
+
+// TestHandleSourceEditSubmit_RenamesArchives proves that a save
+// that keeps the webhook's name renames nothing, and that renaming a
+// webhook renames the archive of each of its database targets and
+// asks nothing of its other targets.
+func TestHandleSourceEditSubmit_RenamesArchives(t *testing.T) {
+	t.Parallel()
+
+	env := setupSourceTest(t)
+	wh := seedWebhookWithRetention(t, env.db, 7)
+	first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+	second := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+	seedTarget(t, env.db, wh.ID, database.TargetTypeLog)
+
+	w := submitEdit(t, env, wh, "")
+	require.Equal(t, http.StatusSeeOther, w.Code)
+	assert.Empty(t, env.archives.Renames())
+
+	wh.Name = renamedWebhookName
+
+	w = submitEdit(t, env, wh, "")
+	require.Equal(t, http.StatusSeeOther, w.Code)
+
+	assert.ElementsMatch(
+		t,
+		[]archiveRename{
+			{first.ID, renamedWebhookName, first.Name},
+			{second.ID, renamedWebhookName, second.Name},
+		},
+		env.archives.Renames(),
+	)
+}
+
+// TestHandleSourceEditSubmit_FailedRenameKeepsTheName proves that a
+// webhook whose archive cannot be renamed keeps its stored name, so
+// the name on disk and the name in the UI do not part, and that the
+// handler puts back what it may already have moved.
+func TestHandleSourceEditSubmit_FailedRenameKeepsTheName(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	env := setupSourceTest(t)
+	wh := seedWebhookWithRetention(t, env.db, 7)
+	tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+
+	env.archives.FailRenames(tgt.ID, errInjectedRename)
+
+	oldName := wh.Name
+	wh.Name = renamedWebhookName
+
+	w := submitEdit(t, env, wh, "")
+	require.Equal(t, http.StatusInternalServerError, w.Code)
+
+	var stored database.Webhook
+
+	require.NoError(
+		t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
+	)
+	assert.Equal(t, oldName, stored.Name)
+
+	assert.Equal(
+		t,
+		[]archiveRename{
+			{tgt.ID, renamedWebhookName, tgt.Name},
+			{tgt.ID, oldName, tgt.Name},
+		},
+		env.archives.Renames(),
+	)
+}
+
+// TestHandleSourceEditSubmit_FailedSaveRenamesBack proves that when
+// the archive is renamed but the new name cannot be saved, the
+// archive is renamed back to the stored name and the stored name
+// stays.
+func TestHandleSourceEditSubmit_FailedSaveRenamesBack(t *testing.T) {
+	t.Parallel()
+
+	env := setupSourceTest(t)
+	wh := seedWebhookWithRetention(t, env.db, 7)
+	tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+
+	failSaveOnTable(t, env.db, "webhooks")
+
+	oldName := wh.Name
+	wh.Name = renamedWebhookName
+
+	w := submitEdit(t, env, wh, "")
+	require.Equal(t, http.StatusInternalServerError, w.Code)
+
+	var stored database.Webhook
+
+	require.NoError(
+		t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
+	)
+	assert.Equal(t, oldName, stored.Name)
+
+	assert.Equal(
+		t,
+		[]archiveRename{
+			{tgt.ID, renamedWebhookName, tgt.Name},
+			{tgt.ID, oldName, tgt.Name},
+		},
+		env.archives.Renames(),
+	)
+}
+
+// errInjectedRead is the failure a test makes reads of the main
+// database report.
+var errInjectedRead = errors.New("injected read failure")
+
+// TestHandleSourceEditSubmit_FailedSaveRenamesBackWithoutReading
+// proves that when the save fails and every later read of the main
+// database fails too, each archive the rename moved is still renamed
+// back: the move back needs no second read of the webhook's targets.
+func TestHandleSourceEditSubmit_FailedSaveRenamesBackWithoutReading(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	env := setupSourceTest(t)
+	wh := seedWebhookWithRetention(t, env.db, 7)
+	first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+	second := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+
+	var saveFailed atomic.Bool
+
+	require.NoError(t, env.db.DB().Callback().Update().
+		Before("gorm:update").
+		Register("test:fail_save", func(tx *gorm.DB) {
+			saveFailed.Store(true)
+
+			_ = tx.AddError(errInjectedSave)
+		}),
+	)
+	require.NoError(t, env.db.DB().Callback().Query().
+		Before("gorm:query").
+		Register("test:fail_reads_after_save", func(tx *gorm.DB) {
+			if saveFailed.Load() {
+				_ = tx.AddError(errInjectedRead)
+			}
+		}),
+	)
+
+	oldName := wh.Name
+	wh.Name = renamedWebhookName
+
+	w := submitEdit(t, env, wh, "")
+	require.Equal(t, http.StatusInternalServerError, w.Code)
+
+	assert.Equal(
+		t,
+		[]archiveRename{
+			{first.ID, renamedWebhookName, first.Name},
+			{second.ID, renamedWebhookName, second.Name},
+			{first.ID, oldName, first.Name},
+			{second.ID, oldName, second.Name},
+		},
+		env.archives.Renames(),
+	)
+}
+
+// TestHandleSourceEditSubmit_EditsDoNotInterleave proves that a second
+// webhook edit submitted while the first is inside its archive rename
+// does not run until the first is saved, so afterwards the stored
+// names are the ones the archive was last renamed to. The stand-in's
+// last rename is the name the file has on disk.
+func TestHandleSourceEditSubmit_EditsDoNotInterleave(t *testing.T) {
+	t.Parallel()
+
+	env := setupSourceTest(t)
+	wh := seedWebhookWithRetention(t, env.db, 7)
+	tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+
+	entered, release := env.archives.BlockNextRename()
+
+	firstEdit, secondEdit := wh, wh
+	firstEdit.Name = "First"
+	secondEdit.Name = "Second"
+
+	firstCode := make(chan int, 1)
+
+	go func() { firstCode <- submitEdit(t, env, firstEdit, "").Code }()
+
+	<-entered
+
+	secondCode := make(chan int, 1)
+
+	go func() { secondCode <- submitEdit(t, env, secondEdit, "").Code }()
+
+	// Were the edits not ordered, the second would run to its end in
+	// this time, while the first is still inside its rename.
+	time.Sleep(200 * time.Millisecond)
+	release()
+
+	assert.Equal(t, http.StatusSeeOther, <-firstCode)
+	assert.Equal(t, http.StatusSeeOther, <-secondCode)
+
+	var (
+		storedWebhook database.Webhook
+		storedTarget  database.Target
+	)
+
+	require.NoError(
+		t, env.db.DB().First(&storedWebhook, "id = ?", wh.ID).Error,
+	)
+	require.NoError(
+		t, env.db.DB().First(&storedTarget, "id = ?", tgt.ID).Error,
+	)
+
+	renames := env.archives.Renames()
+	require.NotEmpty(t, renames)
+	assert.Equal(
+		t,
+		archiveRename{tgt.ID, storedWebhook.Name, storedTarget.Name},
+		renames[len(renames)-1],
+	)
+}
+
+// TestHandleSourceEditSubmit_FailedRenameRenamesTheOthersBack proves
+// that when a webhook has three database targets and only the middle
+// one's archive cannot be renamed, the stored name stays and both
+// others are renamed back, the last one included: the move back does
+// not stop at the target it cannot rename. The handler reaches the
+// targets in the order they were created, which the exact sequence
+// below pins, so the refused target always comes before the last.
+func TestHandleSourceEditSubmit_FailedRenameRenamesTheOthersBack(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	env := setupSourceTest(t)
+	wh := seedWebhookWithRetention(t, env.db, 7)
+	first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+	middle := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+	last := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+
+	env.archives.FailRenames(middle.ID, errNameTaken)
+
+	oldName := wh.Name
+	wh.Name = renamedWebhookName
+
+	w := submitEdit(t, env, wh, "")
+	require.Equal(t, http.StatusConflict, w.Code)
+
+	var stored database.Webhook
+
+	require.NoError(
+		t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
+	)
+	assert.Equal(t, oldName, stored.Name)
+
+	assert.Equal(
+		t,
+		[]archiveRename{
+			{first.ID, renamedWebhookName, first.Name},
+			{middle.ID, renamedWebhookName, middle.Name},
+			{last.ID, renamedWebhookName, last.Name},
+			{first.ID, oldName, first.Name},
+			{middle.ID, oldName, middle.Name},
+			{last.ID, oldName, last.Name},
+		},
+		env.archives.Renames(),
+	)
+}
+
+// TestHandleSourceEditSubmit_ArchiveNameTaken proves that when a file
+// already has an archive's new name, the edit is refused with an
+// error naming that file, and the webhook keeps its stored name.
+func TestHandleSourceEditSubmit_ArchiveNameTaken(t *testing.T) {
+	t.Parallel()
+
+	env := setupSourceTest(t)
+	wh := seedWebhookWithRetention(t, env.db, 7)
+	tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
+
+	env.archives.FailRenames(tgt.ID, errNameTaken)
+
+	oldName := wh.Name
+	wh.Name = renamedWebhookName
+
+	w := submitEdit(t, env, wh, "")
+	require.Equal(t, http.StatusConflict, w.Code)
+	assert.Contains(t, w.Body.String(), "archive-taken.db")
+
+	var stored database.Webhook
+
+	require.NoError(
+		t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
+	)
+	assert.Equal(t, oldName, stored.Name)
+}
+
 // TestSourceEditForm_ForeverWebhookRoundTrips walks the exact path that
 // the removed max="365" cap used to break: render the edit form for a
 // retain-forever webhook, confirm the pre-filled sentinel is not capped