Close the retention follow-ups from the August review (closes #99) #446
@@ -184,16 +184,6 @@ func (r *RetentionReaper) sweep(ctx context.Context) {
|
|||||||
|
|
||||||
wh := webhooks[i]
|
wh := webhooks[i]
|
||||||
|
|
||||||
// Skip retain-forever webhooks before building any query.
|
|
||||||
// RetainsForever covers both the RetentionForeverDays
|
|
||||||
// sentinel and the non-positive values that predate it: the
|
|
||||||
// sentinel is a positive number, so without this the reaper
|
|
||||||
// would compute a cutoff a thousand years in the past and
|
|
||||||
// issue a DELETE matching nothing on every single sweep.
|
|
||||||
if wh.RetainsForever() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nothing to reap if the per-webhook database has never
|
// Nothing to reap if the per-webhook database has never
|
||||||
// been created.
|
// been created.
|
||||||
if !r.dbManager.DBExists(wh.ID) {
|
if !r.dbManager.DBExists(wh.ID) {
|
||||||
@@ -212,6 +202,13 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
webhookID string,
|
webhookID string,
|
||||||
retentionDays int,
|
retentionDays int,
|
||||||
) {
|
) {
|
||||||
|
// A retain-forever webhook has no cutoff, so its database is not
|
||||||
|
// even opened.
|
||||||
|
cutoff, ok := retentionCutoff(time.Now(), retentionDays)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
db, err := r.dbManager.GetDB(webhookID)
|
db, err := r.dbManager.GetDB(webhookID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
r.log.Error(
|
r.log.Error(
|
||||||
@@ -223,11 +220,6 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
cutoff, ok := retentionCutoff(time.Now(), retentionDays)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
deleted, err := reapExpired(ctx, db, cutoff)
|
deleted, err := reapExpired(ctx, db, cutoff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
r.log.Error(
|
r.log.Error(
|
||||||
|
|||||||
@@ -362,7 +362,7 @@ func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
|||||||
t,
|
t,
|
||||||
overflowingRetentionDays,
|
overflowingRetentionDays,
|
||||||
database.RetentionForeverDays,
|
database.RetentionForeverDays,
|
||||||
"the test value must not be rescued by the forever skip",
|
"the test value must not be treated as retain-forever",
|
||||||
)
|
)
|
||||||
|
|
||||||
webhookID := createWebhook(
|
webhookID := createWebhook(
|
||||||
|
|||||||
@@ -41,71 +41,51 @@ type WebhookListItem struct {
|
|||||||
// errMissingURL signals that a required URL was not provided.
|
// errMissingURL signals that a required URL was not provided.
|
||||||
var errMissingURL = errors.New("missing URL")
|
var errMissingURL = errors.New("missing URL")
|
||||||
|
|
||||||
// errInvalidRetention signals a retention_days form value that is not
|
// parseRetentionDays interprets a retention_days form value. It
|
||||||
// a non-negative whole number.
|
// returns the number of days, or, for a value it refuses, the message
|
||||||
var errInvalidRetention = errors.New("invalid retention days")
|
// the create and edit forms show; the message is empty when the value
|
||||||
|
// is accepted.
|
||||||
// errRetentionTooLarge signals a retention_days form value that is a
|
|
||||||
// whole number but larger than the reaper's cutoff arithmetic can
|
|
||||||
// represent. It is distinguished from errInvalidRetention so the form
|
|
||||||
// can tell the user the actual ceiling instead of implying their input
|
|
||||||
// was not a number.
|
|
||||||
var errRetentionTooLarge = errors.New("retention days out of range")
|
|
||||||
|
|
||||||
// retentionErrorMessage returns the message the create and edit forms
|
|
||||||
// show the user for a rejected retention_days value. Any error other
|
|
||||||
// than errRetentionTooLarge falls back to the generic wording, so an
|
|
||||||
// unrecognised parse failure still produces a sensible 400 rather than
|
|
||||||
// an empty alert.
|
|
||||||
func retentionErrorMessage(err error) string {
|
|
||||||
if errors.Is(err, errRetentionTooLarge) {
|
|
||||||
return "Retention must be at most " +
|
|
||||||
strconv.Itoa(database.MaxFiniteRetentionDays) +
|
|
||||||
" days, or 0 to retain events forever."
|
|
||||||
}
|
|
||||||
|
|
||||||
return "Retention must be a whole number of days, or 0 to " +
|
|
||||||
"retain events forever."
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseRetentionDays interprets a retention_days form value.
|
|
||||||
//
|
//
|
||||||
// An empty value yields fallback, which lets the create path apply the
|
// An empty value yields fallback, which lets the create path apply the
|
||||||
// default and the edit path leave the stored value unchanged. A value
|
// default and the edit path leave the stored value unchanged. A value
|
||||||
// of 0 is returned as 0 and is rewritten to the retain-forever
|
// of 0 is returned as 0 and is rewritten to the retain-forever
|
||||||
// sentinel by database.Webhook's BeforeSave hook. Anything unparseable
|
// sentinel by database.Webhook's BeforeSave hook. Anything unparseable
|
||||||
// or negative is an error rather than a silently substituted default.
|
// or negative is refused rather than silently given a default.
|
||||||
//
|
//
|
||||||
// The upper bound is not cosmetic. The reaper computes its cutoff as a
|
// The upper bound is not cosmetic. The reaper computes its cutoff as a
|
||||||
// time.Duration, an int64 nanosecond count, so a day count above
|
// time.Duration, an int64 nanosecond count, so a day count above
|
||||||
// database.MaxFiniteRetentionDays overflows, puts the cutoff in the
|
// database.MaxFiniteRetentionDays overflows, puts the cutoff in the
|
||||||
// future, and deletes every event the webhook has. A finite value
|
// future, and deletes every event the webhook has. A finite value
|
||||||
// above that ceiling is therefore a 400.
|
// above that ceiling is therefore refused, and the message names the
|
||||||
|
// ceiling rather than implying the input was not a number.
|
||||||
//
|
//
|
||||||
// A value at or above the retain-forever sentinel is not out of range:
|
// A value at or above the retain-forever sentinel is not out of range:
|
||||||
// it is what the edit form pre-fills for a retain-forever webhook, so
|
// it is what the edit form pre-fills for a retain-forever webhook, so
|
||||||
// submitting the form back unchanged has to keep meaning "forever"
|
// submitting the form back unchanged has to keep meaning "forever"
|
||||||
// rather than being rejected.
|
// rather than being rejected.
|
||||||
func parseRetentionDays(raw string, fallback int) (int, error) {
|
func parseRetentionDays(raw string, fallback int) (int, string) {
|
||||||
raw = strings.TrimSpace(raw)
|
raw = strings.TrimSpace(raw)
|
||||||
if raw == "" {
|
if raw == "" {
|
||||||
return fallback, nil
|
return fallback, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
v, err := strconv.Atoi(raw)
|
v, err := strconv.Atoi(raw)
|
||||||
if err != nil || v < 0 {
|
if err != nil || v < 0 {
|
||||||
return 0, errInvalidRetention
|
return 0, "Retention must be a whole number of days, or 0 to " +
|
||||||
|
"retain events forever."
|
||||||
}
|
}
|
||||||
|
|
||||||
if v >= database.RetentionForeverDays {
|
if v >= database.RetentionForeverDays {
|
||||||
return database.RetentionForeverDays, nil
|
return database.RetentionForeverDays, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
if v > database.MaxFiniteRetentionDays {
|
if v > database.MaxFiniteRetentionDays {
|
||||||
return 0, errRetentionTooLarge
|
return 0, "Retention must be at most " +
|
||||||
|
strconv.Itoa(database.MaxFiniteRetentionDays) +
|
||||||
|
" days, or 0 to retain events forever."
|
||||||
}
|
}
|
||||||
|
|
||||||
return v, nil
|
return v, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeliveryView is the display-safe projection of a delivery
|
// DeliveryView is the display-safe projection of a delivery
|
||||||
@@ -361,16 +341,13 @@ func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
retentionDays, retErr := parseRetentionDays(
|
retentionDays, errMsg := parseRetentionDays(
|
||||||
retentionStr, database.DefaultRetentionDays,
|
retentionStr, database.DefaultRetentionDays,
|
||||||
)
|
)
|
||||||
if retErr != nil {
|
if errMsg != "" {
|
||||||
h.renderTemplateStatus(
|
h.renderTemplateStatus(
|
||||||
w, r, "sources_new.html",
|
w, r, "sources_new.html",
|
||||||
newSourceFormData(
|
newSourceFormData(errMsg, name, description),
|
||||||
retentionErrorMessage(retErr),
|
|
||||||
name, description,
|
|
||||||
),
|
|
||||||
http.StatusBadRequest,
|
http.StatusBadRequest,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -655,13 +632,13 @@ func (h *Handlers) applyWebhookEdit(
|
|||||||
|
|
||||||
// An empty field falls back to the stored value, so submitting the
|
// An empty field falls back to the stored value, so submitting the
|
||||||
// form without touching retention leaves the policy alone.
|
// form without touching retention leaves the policy alone.
|
||||||
retentionDays, retErr := parseRetentionDays(
|
retentionDays, errMsg := parseRetentionDays(
|
||||||
r.PostFormValue("retention_days"), webhook.RetentionDays,
|
r.PostFormValue("retention_days"), webhook.RetentionDays,
|
||||||
)
|
)
|
||||||
if retErr != nil {
|
if errMsg != "" {
|
||||||
data := map[string]any{
|
data := map[string]any{
|
||||||
tmplKeyWebhook: webhook,
|
tmplKeyWebhook: webhook,
|
||||||
tmplKeyError: retentionErrorMessage(retErr),
|
tmplKeyError: errMsg,
|
||||||
}
|
}
|
||||||
|
|
||||||
h.renderTemplateStatus(w, r, "source_edit.html", data, http.StatusBadRequest)
|
h.renderTemplateStatus(w, r, "source_edit.html", data, http.StatusBadRequest)
|
||||||
|
|||||||
@@ -368,16 +368,24 @@ func TestHandleSourceCreateSubmit_OverflowingRetentionIsRejected(
|
|||||||
// boundary between "too large to represent" and "retain forever": the
|
// boundary between "too large to represent" and "retain forever": the
|
||||||
// sentinel is above MaxFiniteRetentionDays, but it is the value the
|
// sentinel is above MaxFiniteRetentionDays, but it is the value the
|
||||||
// edit form pre-fills, so it must be accepted rather than rejected as
|
// edit form pre-fills, so it must be accepted rather than rejected as
|
||||||
// out of range.
|
// out of range. A value above the sentinel is stored as the sentinel.
|
||||||
func TestHandleSourceCreateSubmit_SentinelIsAcceptedAsForever(
|
func TestHandleSourceCreateSubmit_SentinelIsAcceptedAsForever(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
for _, days := range []int{
|
||||||
sentinel := strconv.Itoa(database.RetentionForeverDays)
|
database.RetentionForeverDays,
|
||||||
|
database.RetentionForeverDays + 1,
|
||||||
|
} {
|
||||||
|
raw := strconv.Itoa(days)
|
||||||
|
|
||||||
w := submitCreate(t, env.handlers, env.cookies, "forever", &sentinel)
|
t.Run(raw, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
|
w := submitCreate(t, env.handlers, env.cookies, "forever", &raw)
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
|
||||||
wh := onlyWebhook(t, env.db)
|
wh := onlyWebhook(t, env.db)
|
||||||
@@ -386,13 +394,16 @@ func TestHandleSourceCreateSubmit_SentinelIsAcceptedAsForever(
|
|||||||
database.RetentionForeverDays,
|
database.RetentionForeverDays,
|
||||||
storedRetentionDays(t, env.db, wh.ID),
|
storedRetentionDays(t, env.db, wh.ID),
|
||||||
)
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput checks that a
|
// TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput checks that a
|
||||||
// validation failure hands the user's typing back, matching what the
|
// validation failure hands the user's typing back, matching what the
|
||||||
// edit form already does. Losing a long description to a mistyped
|
// edit form already does. Losing a long description to a mistyped
|
||||||
// retention value is the kind of thing that makes people give up on a
|
// retention value is the kind of thing that makes people give up on a
|
||||||
// form.
|
// form. Both values carry HTML-special characters, which must come
|
||||||
|
// back escaped rather than as markup.
|
||||||
func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -401,8 +412,8 @@ func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
|||||||
env := setupSourceTest(t)
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
name = "kept-name"
|
name = `kept"><b>name`
|
||||||
description = "a description worth not losing"
|
description = `a </textarea> worth not losing`
|
||||||
)
|
)
|
||||||
|
|
||||||
form := url.Values{}
|
form := url.Values{}
|
||||||
@@ -419,8 +430,10 @@ func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
|||||||
|
|
||||||
body := w.Body.String()
|
body := w.Body.String()
|
||||||
|
|
||||||
assert.Contains(t, body, `value="`+name+`"`)
|
assert.Contains(t, body, `value="kept"><b>name"`)
|
||||||
assert.Contains(t, body, description)
|
assert.Contains(t, body, `a </textarea> worth not losing`)
|
||||||
|
assert.NotContains(t, body, name)
|
||||||
|
assert.NotContains(t, body, description)
|
||||||
}
|
}
|
||||||
|
|
||||||
// submitEdit posts the webhook edit form for the given webhook.
|
// submitEdit posts the webhook edit form for the given webhook.
|
||||||
|
|||||||
Reference in New Issue
Block a user