Name each database target's archive for its webhook and target (closes #376)
check / check (push) Waiting to run
check / check (push) Waiting to run
Each database target now writes its own archive file, archive-WEBHOOKNAME-TARGETNAME-TARGETID.db, named by delivery.ArchiveFileName, in place of one archive per webhook keyed on its UUID. Renaming a webhook or a target renames its archive files (with any -wal and -shm) before the new name is saved, never over an existing file, and moves every one back if a rename or the save fails. The webhook edit, the target edit and target creation share one lock so no two interleave. Deleting a webhook or target leaves its files on disk. Nothing looks for the old archive-WEBHOOKID.db files. The README gives the naming and the recovery steps. Model: opus-5-5
This commit was merged in pull request #418.
This commit is contained in:
@@ -505,6 +505,9 @@ func (h *Handlers) HandleSourceEditSubmit() http.HandlerFunc {
|
||||
|
||||
sourceID := chi.URLParam(r, "sourceID")
|
||||
|
||||
h.renameMu.Lock()
|
||||
defer h.renameMu.Unlock()
|
||||
|
||||
var webhook database.Webhook
|
||||
|
||||
err := h.db.DB().Where(
|
||||
@@ -550,6 +553,7 @@ func (h *Handlers) applyWebhookEdit(
|
||||
return
|
||||
}
|
||||
|
||||
oldName := webhook.Name
|
||||
webhook.Name = name
|
||||
webhook.Description = r.PostFormValue("description")
|
||||
|
||||
@@ -572,8 +576,42 @@ func (h *Handlers) applyWebhookEdit(
|
||||
|
||||
webhook.RetentionDays = retentionDays
|
||||
|
||||
err := h.db.DB().Save(webhook).Error
|
||||
// A new name renames the archive files before it is saved (see
|
||||
// delivery.Engine.Rename). If either step fails, the same targets'
|
||||
// archives go back to the name that is still stored, without
|
||||
// reading the main database again.
|
||||
targets, err := h.renameWebhookArchives(
|
||||
webhook.ID, oldName, webhook.Name,
|
||||
)
|
||||
if err == nil {
|
||||
err = h.db.DB().Save(webhook).Error
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
restoreErr := h.renameArchives(targets, oldName)
|
||||
if restoreErr != nil {
|
||||
h.log.Error(
|
||||
"failed to rename archives back",
|
||||
"webhook_id", webhook.ID,
|
||||
"error", restoreErr,
|
||||
)
|
||||
}
|
||||
|
||||
if errors.Is(err, delivery.ErrArchiveNameTaken) {
|
||||
data := map[string]any{
|
||||
tmplKeyWebhook: webhook,
|
||||
tmplKeyError: "Not saved: " + err.Error() +
|
||||
". Move that archive out of the data directory, " +
|
||||
"its .db together with any -wal and -shm beside " +
|
||||
"it, then save again.",
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusConflict)
|
||||
h.renderTemplate(w, r, "source_edit.html", data)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
h.serverError(w, r, "failed to update webhook", err)
|
||||
|
||||
return
|
||||
@@ -711,11 +749,11 @@ func (h *Handlers) commitWebhookDeletion(
|
||||
return tx.Commit().Error
|
||||
}
|
||||
|
||||
// evictArchiveWriter asks the delivery engine to drop its
|
||||
// cached archive writer for a webhook, closing the archive file
|
||||
// handle.
|
||||
// evictArchiveWriter asks the delivery engine to drop the cached
|
||||
// archive writers of a webhook's database targets, closing their
|
||||
// archive file handles.
|
||||
//
|
||||
// The archive database file is NOT deleted. Unlike the event
|
||||
// The archive database files are NOT deleted. Unlike the event
|
||||
// database — which is per-webhook working storage and is
|
||||
// hard-deleted with the webhook — an archive is explicitly
|
||||
// long-term storage that an operator may want to keep or move
|
||||
@@ -723,50 +761,72 @@ func (h *Handlers) commitWebhookDeletion(
|
||||
// deleting a webhook would be a surprising and unrecoverable
|
||||
// data loss, so the file is left for the operator to handle.
|
||||
func (h *Handlers) evictArchiveWriter(webhookID string) {
|
||||
if h.evictor == nil {
|
||||
if h.archives == nil {
|
||||
return
|
||||
}
|
||||
|
||||
h.evictor.EvictWebhook(webhookID)
|
||||
h.archives.EvictWebhook(webhookID)
|
||||
}
|
||||
|
||||
// evictArchiveWriterIfUnused releases a webhook's archive
|
||||
// writer once the webhook has no database target left to feed
|
||||
// it.
|
||||
//
|
||||
// It is called after any child resource of a webhook is
|
||||
// deleted, and is correct without knowing which kind was: it
|
||||
// evicts only when no database target remains, so deleting one
|
||||
// of several database targets — or deleting an unrelated
|
||||
// target type — leaves a still-needed writer alone. When no
|
||||
// database target ever existed there is no writer and eviction
|
||||
// is a no-op. Soft-deleted targets are excluded by GORM's
|
||||
// default scope, so the row just deleted is not counted.
|
||||
func (h *Handlers) evictArchiveWriterIfUnused(webhookID string) {
|
||||
var remaining int64
|
||||
// evictTargetArchiveWriter is evictArchiveWriter for one deleted
|
||||
// target, and leaves its archive file on disk for the same reason.
|
||||
// A target that is not a database target has no writer, and
|
||||
// evicting it does nothing.
|
||||
func (h *Handlers) evictTargetArchiveWriter(targetID string) {
|
||||
if h.archives == nil {
|
||||
return
|
||||
}
|
||||
|
||||
h.archives.EvictTarget(targetID)
|
||||
}
|
||||
|
||||
// renameWebhookArchives renames the archive file of every database
|
||||
// target of a webhook from the webhook name oldName to newName,
|
||||
// keeping each target's own name. It does nothing when the name is
|
||||
// unchanged. It returns the targets it read, so that a failed edit can
|
||||
// move those same archives back with renameArchives.
|
||||
func (h *Handlers) renameWebhookArchives(
|
||||
webhookID, oldName, newName string,
|
||||
) ([]database.Target, error) {
|
||||
if h.archives == nil || oldName == newName {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var targets []database.Target
|
||||
|
||||
err := h.db.DB().
|
||||
Model(&database.Target{}).
|
||||
Where(
|
||||
"webhook_id = ? AND type = ?",
|
||||
webhookID, database.TargetTypeDatabase,
|
||||
).
|
||||
Count(&remaining).Error
|
||||
Find(&targets).Error
|
||||
if err != nil {
|
||||
h.log.Error(
|
||||
"failed to count remaining database targets",
|
||||
"webhook_id", webhookID,
|
||||
"error", err,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return targets, h.renameArchives(targets, newName)
|
||||
}
|
||||
|
||||
// renameArchives renames the archive file of each of the given
|
||||
// database targets to the webhook name webhookName, keeping each
|
||||
// target's own name. It tries every target even after one fails, so
|
||||
// that moving the archives back after a failed edit leaves none under
|
||||
// the new name, and returns every failure joined.
|
||||
func (h *Handlers) renameArchives(
|
||||
targets []database.Target, webhookName string,
|
||||
) error {
|
||||
var errs []error
|
||||
|
||||
for i := range targets {
|
||||
err := h.archives.Rename(
|
||||
targets[i].ID, webhookName, targets[i].Name,
|
||||
)
|
||||
|
||||
return
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
if remaining > 0 {
|
||||
return
|
||||
}
|
||||
|
||||
h.evictArchiveWriter(webhookID)
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// ownedWebhook resolves the request's sourceID parameter to a
|
||||
@@ -1263,6 +1323,9 @@ func (h *Handlers) HandleTargetCreate() http.HandlerFunc {
|
||||
|
||||
sourceID := chi.URLParam(r, "sourceID")
|
||||
|
||||
h.renameMu.Lock()
|
||||
defer h.renameMu.Unlock()
|
||||
|
||||
var webhook database.Webhook
|
||||
|
||||
err := h.db.DB().Where(
|
||||
@@ -1639,29 +1702,28 @@ func (h *Handlers) HandleEntrypointDelete() http.HandlerFunc {
|
||||
)
|
||||
}
|
||||
|
||||
// HandleTargetDelete handles deleting a target. Deleting the
|
||||
// last database target of a webhook leaves its archive writer
|
||||
// with nothing to write, so the writer is evicted and its
|
||||
// handle closed; the archive file is left on disk.
|
||||
// HandleTargetDelete handles deleting a target. A deleted
|
||||
// database target's archive writer is evicted and its handle
|
||||
// closed; the archive file is left on disk.
|
||||
func (h *Handlers) HandleTargetDelete() http.HandlerFunc {
|
||||
return h.deleteChildResource(
|
||||
"targetID", &database.Target{},
|
||||
"failed to delete target",
|
||||
h.evictArchiveWriterIfUnused,
|
||||
h.evictTargetArchiveWriter,
|
||||
targetDeleted,
|
||||
)
|
||||
}
|
||||
|
||||
// deleteChildResource returns a handler that deletes a child
|
||||
// resource (entrypoint or target) belonging to a webhook. The
|
||||
// optional afterDelete hook runs with the webhook's id once the
|
||||
// delete has succeeded, before the redirect, which carries done as
|
||||
// optional afterDelete hook runs with the child's id once the
|
||||
// delete has removed it, before the redirect, which carries done as
|
||||
// its notice.
|
||||
func (h *Handlers) deleteChildResource(
|
||||
idParam string,
|
||||
model any,
|
||||
errMsg string,
|
||||
afterDelete func(webhookID string),
|
||||
afterDelete func(childID string),
|
||||
done noticeCode,
|
||||
) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1698,8 +1760,10 @@ func (h *Handlers) deleteChildResource(
|
||||
return
|
||||
}
|
||||
|
||||
if afterDelete != nil {
|
||||
afterDelete(webhook.ID)
|
||||
// Only for a row this webhook really had: the id came from
|
||||
// the URL and may name another webhook's child.
|
||||
if afterDelete != nil && result.RowsAffected > 0 {
|
||||
afterDelete(childID)
|
||||
}
|
||||
|
||||
http.Redirect(
|
||||
|
||||
Reference in New Issue
Block a user