notify: a reply other than 2xx is a failed delivery (closes #287)
check / check (push) Canceled after 0s

A notification endpoint that answered with a redirect was counted as
delivered: the request is not sent through a client, so the redirect
is never followed, and only a status of 400 or more counted as a
failure. Now any status outside 200-299 fails the delivery for ntfy,
Slack and Mattermost, so it is retried and logged like the others.

Model: opus-5-5
This commit is contained in:
2026-10-15 12:52:31 +00:00
parent 5e2c456e85
commit cf63fdea6c
4 changed files with 72 additions and 9 deletions
+1 -1
View File
@@ -285,7 +285,7 @@ includes:
Each endpoint is sent each notification on its own, in the background. A Each endpoint is sent each notification on its own, in the background. A
delivery that fails (a network error, no reply within 10 seconds, or an HTTP delivery that fails (a network error, no reply within 10 seconds, or an HTTP
status of 400 or more) is retried up to 5 times: the first retry after about 1 status other than 2xx) is retried up to 5 times: the first retry after about 1
second, each wait after that twice as long up to 60 seconds, every wait varied second, each wait after that twice as long up to 60 seconds, every wait varied
at random by up to 25%. A delivery still failing after that is logged and at random by up to 25%. A delivery still failing after that is logged and
dropped. dropped.
+2
View File
@@ -19,6 +19,8 @@ milestone: https://git.eeqj.de/sneak/dnswatcher/milestone/7
# Completed Steps # Completed Steps
- 2026-10-15: a notification endpoint that answers with a redirect, or any
status other than 2xx, is a failed delivery, retried and logged (closes #287).
- 2026-10-15: the log names a notification endpoint (`slack`, `mattermost`, - 2026-10-15: the log names a notification endpoint (`slack`, `mattermost`,
`ntfy`) and never shows its URL's path or query (closes #288). `ntfy`) and never shows its URL's path or query (closes #288).
- 2026-10-13: a hostname or a domain whose zone's nameservers all stop answering - 2026-10-13: a hostname or a domain whose zone's nameservers all stop answering
+11 -8
View File
@@ -25,9 +25,6 @@ import (
// HTTP client timeout. // HTTP client timeout.
const httpClientTimeout = 10 * time.Second const httpClientTimeout = 10 * time.Second
// HTTP status code thresholds.
const httpStatusClientError = 400
// Sentinel errors for notification failures. // Sentinel errors for notification failures.
var ( var (
// ErrNtfyFailed indicates the ntfy request failed. // ErrNtfyFailed indicates the ntfy request failed.
@@ -344,7 +341,10 @@ func (svc *Service) sendNtfy(
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
if resp.StatusCode >= httpStatusClientError { // Only a 2xx reply means ntfy took the notification. A
// redirect is not followed, so it fails like any other.
if resp.StatusCode < http.StatusOK ||
resp.StatusCode >= http.StatusMultipleChoices {
return fmt.Errorf( return fmt.Errorf(
"%w: status %d", "%w: status %d",
ErrNtfyFailed, resp.StatusCode, ErrNtfyFailed, resp.StatusCode,
@@ -385,9 +385,9 @@ type SlackAttachment struct {
// sendSlack posts to a Slack or Mattermost incoming webhook, which // sendSlack posts to a Slack or Mattermost incoming webhook, which
// take the same payload. endpoint, "slack" or "mattermost", names // take the same payload. endpoint, "slack" or "mattermost", names
// the webhook in the log in place of its URL, which lets anyone who // the webhook in the log in place of its URL, which lets anyone who
// has it post to the channel. An HTTP error status is returned // has it post to the channel. A reply status other than 2xx is
// wrapped in failed, ErrSlackFailed or ErrMattermostFailed, so the // returned wrapped in failed, ErrSlackFailed or ErrMattermostFailed,
// error names the endpoint. // so the error names the endpoint.
func (svc *Service) sendSlack( func (svc *Service) sendSlack(
ctx context.Context, ctx context.Context,
webhookURL *url.URL, webhookURL *url.URL,
@@ -437,7 +437,10 @@ func (svc *Service) sendSlack(
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
if resp.StatusCode >= httpStatusClientError { // Only a 2xx reply means the webhook took the notification. A
// redirect is not followed, so it fails like any other.
if resp.StatusCode < http.StatusOK ||
resp.StatusCode >= http.StatusMultipleChoices {
return fmt.Errorf( return fmt.Errorf(
"%w: status %d", "%w: status %d",
failed, resp.StatusCode, failed, resp.StatusCode,
+58
View File
@@ -4,9 +4,12 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"maps"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url" "net/url"
"slices"
"strings"
"sync" "sync"
"sync/atomic" "sync/atomic"
"testing" "testing"
@@ -536,3 +539,58 @@ func TestSendNotificationPermanentFailureLogsError(
return attempts >= 3 return attempts >= 3
}) })
} }
// TestSendNotificationRedirectFails checks that a 302 reply is a
// failed delivery at each endpoint: the server is sent the first
// attempt and the retry, and each endpoint logs a retry line and
// then the final failure line, both naming the 302.
func TestSendNotificationRedirectFails(t *testing.T) {
t.Parallel()
var requests atomic.Int64
srv := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
http.Redirect(w, r, "/moved", http.StatusFound)
}),
)
defer srv.Close()
logs := sendToEachEndpoint(t, srv.URL, "t")
// ntfy, Slack and Mattermost each send once and retry once.
const wantRequests = 3 * 2
if sent := requests.Load(); sent != wantRequests {
t.Errorf("server got %d requests, want %d", sent, wantRequests)
}
// For each endpoint, the messages of its lines whose error names
// the 302, in the order they were logged.
got := map[string][]string{}
for _, record := range parseLog(t, logs) {
if strings.Contains(record.Error, "status 302") {
got[record.Endpoint] = append(
got[record.Endpoint], record.Msg,
)
}
}
lines := []string{
"notification delivery failed, retrying",
"failed to send notification after retries",
}
want := map[string][]string{
endpointNtfy: lines,
endpointSlack: lines,
endpointMattermost: lines,
}
if !maps.EqualFunc(got, want, slices.Equal) {
t.Errorf("lines whose error names the 302 = %v, want %v",
got, want)
}
}