Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
85b71b1d70 |
@@ -149,11 +149,9 @@ same-origin path `/api/v1/reports`: a random per-browser `clientId` kept in
|
|||||||
`localStorage`, `geo` sent as null, and each host's unreported, non-paused
|
`localStorage`, `geo` sent as null, and each host's unreported, non-paused
|
||||||
samples (timestamp, latency, error). A per-host high-water mark makes every
|
samples (timestamp, latency, error). A per-host high-water mark makes every
|
||||||
report a delta, so only new samples are sent; the mark advances only on a
|
report a delta, so only new samples are sent; the mark advances only on a
|
||||||
delivered report, and while paused nothing is sent. Changing the interval clears
|
delivered report, and while paused nothing is sent. Delivery failure is quiet —
|
||||||
each host's history on the page; the samples in it not yet reported still go in
|
one debug-log line per outage, retried at the next interval, never blocking
|
||||||
the next report. Delivery failure is quiet — one debug-log line per outage,
|
probing. The report-building step is a pure function of host state.
|
||||||
retried at the next interval, never blocking probing. The report-building step
|
|
||||||
is a pure function of host state.
|
|
||||||
|
|
||||||
### Backend
|
### Backend
|
||||||
|
|
||||||
@@ -231,15 +229,9 @@ Files are created with `O_EXCL` to prevent overwrites.
|
|||||||
#### Report limits
|
#### Report limits
|
||||||
|
|
||||||
`POST /api/v1/reports` takes reports from anyone who can reach it, without
|
`POST /api/v1/reports` takes reports from anyone who can reach it, without
|
||||||
credentials, so it is bounded instead. Each refusal below answers with the same
|
credentials, so it is bounded instead. Both refusals below answer with the same
|
||||||
`{"status":"error"}` body as any other error.
|
`{"status":"error"}` body as any other error.
|
||||||
|
|
||||||
- **JSON only.** A report must be sent with `Content-Type: application/json`,
|
|
||||||
parameters such as `charset` allowed, as the page sends it; any other is
|
|
||||||
refused with 415, and nothing of it is stored. A page on another origin can
|
|
||||||
send a `text/plain` POST, or a form, without a CORS preflight; refusing them
|
|
||||||
keeps any site from making its visitors' browsers store reports (see
|
|
||||||
[CORS](#cors)).
|
|
||||||
- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may
|
- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may
|
||||||
send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with
|
send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with
|
||||||
`Retry-After: 60`. The minute slides: reports from the minute before still
|
`Retry-After: 60`. The minute slides: reports from the minute before still
|
||||||
@@ -412,10 +404,9 @@ address nginx passes on and no other. The image:
|
|||||||
|
|
||||||
- Listens on port 8080 by default (override with `PORT` env var)
|
- Listens on port 8080 by default (override with `PORT` env var)
|
||||||
- Takes the client address from `X-Forwarded-For` only on requests from the
|
- Takes the client address from `X-Forwarded-For` only on requests from the
|
||||||
reverse proxies named in `TRUSTED_PROXIES`, by default from any address in
|
reverse proxies named in `TRUSTED_PROXIES`, and by default from none.
|
||||||
`10.0.0.0/8`, `172.16.0.0/12` or `192.168.0.0/16`. `bin/entrypoint.sh` checks
|
`bin/entrypoint.sh` checks each entry with `netwatch-server check-cidr`, which
|
||||||
each entry with `netwatch-server check-cidr`, which parses it as the backend
|
parses it as the backend parses its own `TRUSTED_PROXIES`
|
||||||
parses its own `TRUSTED_PROXIES`
|
|
||||||
- Sends access logs to stdout
|
- Sends access logs to stdout
|
||||||
- Caches static assets with immutable headers
|
- Caches static assets with immutable headers
|
||||||
- Sends the security headers `REPO_POLICIES.md` requires on every response, as
|
- Sends the security headers `REPO_POLICIES.md` requires on every response, as
|
||||||
@@ -434,9 +425,9 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for netwatch needs:
|
|||||||
|
|
||||||
- **Port:** container port `8080`.
|
- **Port:** container port `8080`.
|
||||||
- **Volume:** container path `/data`; the reports are kept in `/data/reports`.
|
- **Volume:** container path `/data`; the reports are kept in `/data/reports`.
|
||||||
- **Environment variables:** none is required. An empty one counts as unset,
|
- **Environment variables:** none is required. An empty one counts as unset, and
|
||||||
except `TRUSTED_PROXIES`, and one set to a value netwatch cannot use stops the
|
one set to a value netwatch cannot use stops the container at start, with the
|
||||||
container at start, with the reason in its log.
|
reason in its log.
|
||||||
- `PORT`, default `8080`: the container port, from 1 to 65535. `8081` cannot
|
- `PORT`, default `8080`: the container port, from 1 to 65535. `8081` cannot
|
||||||
be used: the backend listens on it inside the container
|
be used: the backend listens on it inside the container
|
||||||
- `REPORTS_PER_MINUTE`, default `60`: reports each client address may send a
|
- `REPORTS_PER_MINUTE`, default `60`: reports each client address may send a
|
||||||
@@ -450,17 +441,16 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for netwatch needs:
|
|||||||
in: `/data` or a path below it, with no `.` or `..` part and no extra `/`.
|
in: `/data` or a path below it, with no `.` or `..` part and no extra `/`.
|
||||||
The container also stops if the path goes through a symbolic link that
|
The container also stops if the path goes through a symbolic link that
|
||||||
leads out of `/data` or is written as a full path
|
leads out of `/data` or is written as a full path
|
||||||
- `TRUSTED_PROXIES`, default `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`: the
|
- `TRUSTED_PROXIES`, default empty: set it to the address the reverse proxy
|
||||||
addresses the reverse proxy in front of the container connects from, as IP
|
in front of the container connects from, as an IP address or CIDR; several
|
||||||
addresses or CIDRs separated by commas. nginx takes the client address
|
are separated by commas. nginx takes the client address from
|
||||||
from `X-Forwarded-For` only on a request from one of them, and the rate
|
`X-Forwarded-For` only on a request from one of them, and the rate limit
|
||||||
limit counts that address. A list replaces the default. Set and empty,
|
counts that address. Unset, `X-Forwarded-For` is ignored and every client
|
||||||
`X-Forwarded-For` is ignored and every client behind the proxy shares the
|
behind the proxy shares the proxy's one allowance of `REPORTS_PER_MINUTE`.
|
||||||
proxy's one allowance of `REPORTS_PER_MINUTE`. Any client that connects
|
Name only addresses nothing but the proxy connects from: any client that
|
||||||
from one of these addresses can write its own `X-Forwarded-For`, and
|
connects from one can write its own `X-Forwarded-For`, and through a port
|
||||||
through a port Docker publishes, every client may connect from the Docker
|
Docker publishes, every client may connect from the Docker network's
|
||||||
network's gateway, such as `172.17.0.1`, which the default includes; where
|
gateway, such as `172.17.0.1`.
|
||||||
that matters, name only the addresses nothing but the proxy connects from.
|
|
||||||
- `METRICS_USERNAME` and `METRICS_PASSWORD`, default empty: with both set,
|
- `METRICS_USERNAME` and `METRICS_PASSWORD`, default empty: with both set,
|
||||||
the backend records Prometheus metrics of its requests and serves them at
|
the backend records Prometheus metrics of its requests and serves them at
|
||||||
`/metrics` on the container port, to requests with this user name and
|
`/metrics` on the container port, to requests with this user name and
|
||||||
|
|||||||
@@ -27,35 +27,6 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
|||||||
is still the first middleware; `Write500OnPanic`, after the request log and
|
is still the first middleware; `Write500OnPanic`, after the request log and
|
||||||
after the metrics, writes the 500 as the panic passes and lets it go on to
|
after the metrics, writes the 500 as the panic passes and lets it go on to
|
||||||
Sentry and `Recoverer`, which now writes a 500 only when no status was written
|
Sentry and `Recoverer`, which now writes a 500 only when no status was written
|
||||||
- 2026-10-14: with `TRUSTED_PROXIES` unset, the container's nginx trusts the
|
|
||||||
three private ranges of RFC 1918, `10.0.0.0/8`, `172.16.0.0/12` and
|
|
||||||
`192.168.0.0/16` ([#119](https://git.eeqj.de/sneak/netwatch/issues/119)), as
|
|
||||||
sneak ruled on 2026-09-22 for every app; it trusted no proxy before. A list
|
|
||||||
replaces them, and set and empty it trusts none. The backend inside the
|
|
||||||
container still trusts only `127.0.0.1/32`
|
|
||||||
- 2026-10-14: `POST /api/v1/reports` takes only a report sent with
|
|
||||||
`Content-Type: application/json`, parameters such as `charset` allowed, and
|
|
||||||
refuses any other with 415 and the usual error body, storing nothing
|
|
||||||
([#126](https://git.eeqj.de/sneak/netwatch/issues/126)). A page on another
|
|
||||||
origin can send a `text/plain` POST without a CORS preflight, so any site
|
|
||||||
could make its visitors' browsers store reports; a JSON POST waits for the
|
|
||||||
preflight, which only an origin in `CORS_ALLOWED_ORIGINS` passes. The page
|
|
||||||
already sent `application/json`. A test posts a report as `text/plain`, as
|
|
||||||
`application/x-www-form-urlencoded`, as `multipart/form-data`, with no
|
|
||||||
`Content-Type`, and as `application/json` with and without a charset, and
|
|
||||||
checks which are stored
|
|
||||||
- 2026-10-14: changing the interval no longer drops the samples not yet reported
|
|
||||||
to the backend ([#124](https://git.eeqj.de/sneak/netwatch/issues/124)). The
|
|
||||||
interval menu resets each host with the new `HostState.reset`, which moves the
|
|
||||||
history's entries to `clearedHistory`, keeping the last 100 of all it has
|
|
||||||
cleared, and `buildReport` reads both, so the next report has the samples
|
|
||||||
recorded before the change that no report has delivered. Until its next check,
|
|
||||||
such a host is sent with the status of its last check, not `pending`. The
|
|
||||||
page's history still starts over, and the per-host marks still leave out of a
|
|
||||||
report the samples a delivered one had. Unit tests reset a host as the
|
|
||||||
interval menu does and find its unreported samples in the next report, and
|
|
||||||
only the last 100 after more were cleared; `web/src/main.js` now exports
|
|
||||||
`Reporter` for them
|
|
||||||
- 2026-10-14: the repo has the layout `REPO_POLICIES.md` gives
|
- 2026-10-14: the repo has the layout `REPO_POLICIES.md` gives
|
||||||
([#30](https://git.eeqj.de/sneak/netwatch/issues/30)). The Go module is at the
|
([#30](https://git.eeqj.de/sneak/netwatch/issues/30)). The Go module is at the
|
||||||
root (`go.mod`, `go.sum`, `.golangci.yml`, `cmd/netwatch-server/`,
|
root (`go.mod`, `go.sum`, `.golangci.yml`, `cmd/netwatch-server/`,
|
||||||
|
|||||||
+4
-6
@@ -34,11 +34,9 @@ fi
|
|||||||
|
|
||||||
# TRUSTED_PROXIES names the reverse proxies in front of the container,
|
# TRUSTED_PROXIES names the reverse proxies in front of the container,
|
||||||
# as IP addresses or CIDRs separated by commas. nginx takes the client
|
# as IP addresses or CIDRs separated by commas. nginx takes the client
|
||||||
# address from X-Forwarded-For only on a request from one of them.
|
# address from X-Forwarded-For only on a request from one of them, so
|
||||||
# Unset, they are the three private ranges of RFC 1918; set and empty,
|
# unset or empty, it trusts no one. configs/nginx.conf includes the file
|
||||||
# nginx trusts no one, which is why the default below follows - and not
|
# written here, one set_real_ip_from line per entry.
|
||||||
# :-. configs/nginx.conf includes the file written here, one
|
|
||||||
# set_real_ip_from line per entry.
|
|
||||||
#
|
#
|
||||||
# nginx looks up an entry it cannot read as an address as a hostname,
|
# nginx looks up an entry it cannot read as an address as a hostname,
|
||||||
# and trusts what it finds (1.2.3 is found as 1.2.0.3). So each entry
|
# and trusts what it finds (1.2.3 is found as 1.2.0.3). So each entry
|
||||||
@@ -47,7 +45,7 @@ fi
|
|||||||
# TRUSTED_PROXIES. Its error, naming the CIDR, is dropped for the one
|
# TRUSTED_PROXIES. Its error, naming the CIDR, is dropped for the one
|
||||||
# below, naming the entry as written. set -f keeps a * in an entry from
|
# below, naming the entry as written. set -f keeps a * in an entry from
|
||||||
# becoming a list of file names.
|
# becoming a list of file names.
|
||||||
TRUSTED_PROXIES="${TRUSTED_PROXIES-10.0.0.0/8,172.16.0.0/12,192.168.0.0/16}"
|
TRUSTED_PROXIES="${TRUSTED_PROXIES:-}"
|
||||||
set -f
|
set -f
|
||||||
for proxy in $(printf '%s' "$TRUSTED_PROXIES" | tr ',' ' '); do
|
for proxy in $(printf '%s' "$TRUSTED_PROXIES" | tr ',' ' '); do
|
||||||
case "$proxy" in
|
case "$proxy" in
|
||||||
|
|||||||
+3
-4
@@ -18,10 +18,9 @@ server {
|
|||||||
|
|
||||||
# The client address comes from X-Forwarded-For only on a request
|
# The client address comes from X-Forwarded-For only on a request
|
||||||
# from the reverse proxies in TRUSTED_PROXIES: bin/entrypoint.sh
|
# from the reverse proxies in TRUSTED_PROXIES: bin/entrypoint.sh
|
||||||
# writes one set_real_ip_from line for each into this file. Unset,
|
# writes one set_real_ip_from line for each into this file, and
|
||||||
# TRUSTED_PROXIES is the three private ranges of RFC 1918; set and
|
# leaves it empty when TRUSTED_PROXIES is unset, so that by default
|
||||||
# empty, the file is empty and the client address is the one each
|
# the client address is the one each request comes from.
|
||||||
# request comes from.
|
|
||||||
include /etc/nginx/trusted-proxies.conf;
|
include /etc/nginx/trusted-proxies.conf;
|
||||||
real_ip_header X-Forwarded-For;
|
real_ip_header X-Forwarded-For;
|
||||||
real_ip_recursive on;
|
real_ip_recursive on;
|
||||||
|
|||||||
@@ -5,9 +5,7 @@ package handlers
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"mime"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"sneak.berlin/go/netwatch/internal/globals"
|
"sneak.berlin/go/netwatch/internal/globals"
|
||||||
@@ -20,10 +18,6 @@ import (
|
|||||||
|
|
||||||
const jsonContentType = "application/json; charset=utf-8"
|
const jsonContentType = "application/json; charset=utf-8"
|
||||||
|
|
||||||
// errNotJSON is what decodeJSON returns for a request whose
|
|
||||||
// Content-Type is not application/json.
|
|
||||||
var errNotJSON = errors.New("content type is not application/json")
|
|
||||||
|
|
||||||
// reportAppender is the subset of the report buffer the handlers
|
// reportAppender is the subset of the report buffer the handlers
|
||||||
// depend on. Defining it here keeps the storage failure path
|
// depend on. Defining it here keeps the storage failure path
|
||||||
// exercisable with a stub in tests.
|
// exercisable with a stub in tests.
|
||||||
@@ -86,25 +80,14 @@ func (s *Handlers) respondJSON(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// decodeJSON decodes the request body into v. It returns errNotJSON,
|
// decodeJSON decodes the request body into v. The body is
|
||||||
// reading nothing, unless the request's Content-Type is
|
// expected to already be bounded by the body-size middleware, so
|
||||||
// application/json, with or without parameters such as charset. The
|
// a caller can distinguish an over-limit body from malformed
|
||||||
// body is expected to already be bounded by the body-size middleware,
|
// JSON by testing the returned error for *http.MaxBytesError.
|
||||||
// so a caller can distinguish an over-limit body from malformed JSON
|
|
||||||
// by testing the returned error for *http.MaxBytesError.
|
|
||||||
func (s *Handlers) decodeJSON(
|
func (s *Handlers) decodeJSON(
|
||||||
_ http.ResponseWriter,
|
_ http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
v any,
|
v any,
|
||||||
) error {
|
) error {
|
||||||
// A page on another origin can POST text/plain or a form without
|
|
||||||
// a CORS preflight, so its visitors' browsers would send it
|
|
||||||
// whatever the CORS settings. A JSON POST waits for the
|
|
||||||
// preflight, so only JSON is taken.
|
|
||||||
mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
|
||||||
if err != nil || mediaType != "application/json" {
|
|
||||||
return errNotJSON
|
|
||||||
}
|
|
||||||
|
|
||||||
return json.NewDecoder(r.Body).Decode(v)
|
return json.NewDecoder(r.Body).Decode(v)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ type report struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// HandleReport returns a handler that accepts telemetry
|
// HandleReport returns a handler that accepts telemetry
|
||||||
// reports from NetWatch clients, sent as application/json.
|
// reports from NetWatch clients.
|
||||||
func (s *Handlers) HandleReport() http.HandlerFunc {
|
func (s *Handlers) HandleReport() http.HandlerFunc {
|
||||||
type response struct {
|
type response struct {
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
@@ -66,15 +66,9 @@ func (s *Handlers) HandleReport() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// decodeErrorStatus logs a report decode failure and returns the
|
// decodeErrorStatus logs a report decode failure and returns the
|
||||||
// status to send: 415 when the report was not sent as JSON, 413 when
|
// status to send: 413 when the body exceeded the size limit,
|
||||||
// the body exceeded the size limit, otherwise 400 for malformed JSON.
|
// otherwise 400 for malformed JSON.
|
||||||
func (s *Handlers) decodeErrorStatus(err error) int {
|
func (s *Handlers) decodeErrorStatus(err error) int {
|
||||||
if errors.Is(err, errNotJSON) {
|
|
||||||
s.log.Warn("report refused: Content-Type is not application/json")
|
|
||||||
|
|
||||||
return http.StatusUnsupportedMediaType
|
|
||||||
}
|
|
||||||
|
|
||||||
var tooLarge *http.MaxBytesError
|
var tooLarge *http.MaxBytesError
|
||||||
if errors.As(err, &tooLarge) {
|
if errors.As(err, &tooLarge) {
|
||||||
s.log.Warn("report body too large", "limit_bytes", tooLarge.Limit)
|
s.log.Warn("report body too large", "limit_bytes", tooLarge.Limit)
|
||||||
|
|||||||
@@ -27,17 +27,6 @@ type stubAppender struct {
|
|||||||
|
|
||||||
func (s stubAppender) Append(any) error { return s.err }
|
func (s stubAppender) Append(any) error { return s.err }
|
||||||
|
|
||||||
// recordingAppender keeps every report the handler stores.
|
|
||||||
type recordingAppender struct {
|
|
||||||
reports []any
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *recordingAppender) Append(v any) error {
|
|
||||||
a.reports = append(a.reports, v)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTestHandlers(buf stubAppender, out io.Writer) *handlers.Handlers {
|
func newTestHandlers(buf stubAppender, out io.Writer) *handlers.Handlers {
|
||||||
return handlers.NewForTest(buf, slog.New(slog.NewJSONHandler(out, nil)))
|
return handlers.NewForTest(buf, slog.New(slog.NewJSONHandler(out, nil)))
|
||||||
}
|
}
|
||||||
@@ -94,7 +83,6 @@ func TestHandleReportAcceptsValidReports(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(tt.body),
|
strings.NewReader(tt.body),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -115,89 +103,6 @@ func TestHandleReportAcceptsValidReports(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleReportStoresOnlyJSON checks that a report is stored only
|
|
||||||
// when sent as application/json, as the page sends it. One sent as
|
|
||||||
// text/plain or as a form, which a page on another origin can send
|
|
||||||
// without a CORS preflight, is refused with 415.
|
|
||||||
func TestHandleReportStoresOnlyJSON(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
contentType string
|
|
||||||
wantStatus int
|
|
||||||
wantStored int
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "text/plain",
|
|
||||||
contentType: "text/plain;charset=UTF-8",
|
|
||||||
wantStatus: http.StatusUnsupportedMediaType,
|
|
||||||
wantStored: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a urlencoded form",
|
|
||||||
contentType: "application/x-www-form-urlencoded",
|
|
||||||
wantStatus: http.StatusUnsupportedMediaType,
|
|
||||||
wantStored: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a multipart form",
|
|
||||||
contentType: "multipart/form-data; boundary=x",
|
|
||||||
wantStatus: http.StatusUnsupportedMediaType,
|
|
||||||
wantStored: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no Content-Type",
|
|
||||||
contentType: "",
|
|
||||||
wantStatus: http.StatusUnsupportedMediaType,
|
|
||||||
wantStored: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "application/json",
|
|
||||||
contentType: "application/json",
|
|
||||||
wantStatus: http.StatusOK,
|
|
||||||
wantStored: 1,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "application/json with a charset",
|
|
||||||
contentType: "application/json;charset=UTF-8",
|
|
||||||
wantStatus: http.StatusOK,
|
|
||||||
wantStored: 1,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var stored recordingAppender
|
|
||||||
|
|
||||||
h := handlers.NewForTest(&stored, slog.New(slog.DiscardHandler))
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(),
|
|
||||||
http.MethodPost, "/api/v1/reports",
|
|
||||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
|
||||||
)
|
|
||||||
|
|
||||||
if tt.contentType != "" {
|
|
||||||
req.Header.Set("Content-Type", tt.contentType)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != tt.wantStatus {
|
|
||||||
t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(stored.reports) != tt.wantStored {
|
|
||||||
t.Errorf("reports stored = %d, want %d",
|
|
||||||
len(stored.reports), tt.wantStored)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandleReportStorageFailureIsNon2xx(t *testing.T) {
|
func TestHandleReportStorageFailureIsNon2xx(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -208,7 +113,6 @@ func TestHandleReportStorageFailureIsNon2xx(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -234,7 +138,6 @@ func TestHandleReportFullIs507(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -258,7 +161,6 @@ func TestHandleReportMalformedJSONIs400(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{not json`),
|
strings.NewReader(`{not json`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -283,8 +185,6 @@ func TestHandleReportOversizeIs413(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"clientId":"`+strings.Repeat("x", 200)+`"}`),
|
strings.NewReader(`{"clientId":"`+strings.Repeat("x", 200)+`"}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
// No declared length, so only the middleware's read cap can
|
// No declared length, so only the middleware's read cap can
|
||||||
// stop this body.
|
// stop this body.
|
||||||
req.ContentLength = -1
|
req.ContentLength = -1
|
||||||
@@ -313,7 +213,6 @@ func TestHandleReportDoesNotLogRawGeo(t *testing.T) {
|
|||||||
`{"clientId":"c1","geo":{"raw":"`+sentinel+`"},"hosts":[]}`,
|
`{"clientId":"c1","geo":{"raw":"`+sentinel+`"},"hosts":[]}`,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -346,7 +245,6 @@ func TestHandleReportLogsClientIDCutToBound(t *testing.T) {
|
|||||||
`{"clientId":"`+long+`","timestamp":"`+long+`","hosts":[]}`,
|
`{"clientId":"`+long+`","timestamp":"`+long+`","hosts":[]}`,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -386,7 +284,6 @@ func TestHandleReportDecodeErrorLogIsBounded(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"hosts":[{"history":[{"t":`+huge+`}]}]}`),
|
strings.NewReader(`{"hosts":[{"history":[{"t":`+huge+`}]}]}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
|||||||
@@ -66,7 +66,6 @@ func TestReportsAreRateLimited(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
srv.ServeHTTP(rec, req)
|
srv.ServeHTTP(rec, req)
|
||||||
|
|
||||||
return rec.Code
|
return rec.Code
|
||||||
|
|||||||
@@ -64,8 +64,8 @@ main() {
|
|||||||
# configs/nginx.conf is a template: the image renders it over its own
|
# configs/nginx.conf is a template: the image renders it over its own
|
||||||
# default.conf, with the same port and limit bin/entrypoint.sh uses.
|
# default.conf, with the same port and limit bin/entrypoint.sh uses.
|
||||||
# The empty file it includes trusts no proxy, as bin/entrypoint.sh
|
# The empty file it includes trusts no proxy, as bin/entrypoint.sh
|
||||||
# writes it when TRUSTED_PROXIES is set and empty. It also includes
|
# writes it when TRUSTED_PROXIES is unset. It also includes the
|
||||||
# the security headers, so the page runs under the shipped policy.
|
# security headers, so the page runs under the shipped policy.
|
||||||
docker run -d --rm --name "$SERVER" \
|
docker run -d --rm --name "$SERVER" \
|
||||||
--network "$NETWORK" --network-alias netwatch \
|
--network "$NETWORK" --network-alias netwatch \
|
||||||
-e PORT=8080 -e NGINX_ENVSUBST_FILTER='^PORT$' \
|
-e PORT=8080 -e NGINX_ENVSUBST_FILTER='^PORT$' \
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import {
|
|||||||
latencyClass,
|
latencyClass,
|
||||||
latencyHex,
|
latencyHex,
|
||||||
measureLatency,
|
measureLatency,
|
||||||
Reporter,
|
|
||||||
tick,
|
tick,
|
||||||
} from "../../web/src/main.js";
|
} from "../../web/src/main.js";
|
||||||
|
|
||||||
@@ -379,98 +378,6 @@ test("a check that fails in the first round, which is discarded, writes nothing
|
|||||||
assert.deepEqual(consoleLines("info"), []);
|
assert.deepEqual(consoleLines("info"), []);
|
||||||
});
|
});
|
||||||
|
|
||||||
// The reports Reporter sent in this test, each as the hosts it sent.
|
|
||||||
function reportedHosts() {
|
|
||||||
return fetch.mock.calls.map(
|
|
||||||
(call) => JSON.parse(call.arguments[1].body).hosts,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// In the next two tests every report is delivered, and only Target has
|
|
||||||
// checks to report. target.reset() does to Target what the interval menu
|
|
||||||
// does to each target.
|
|
||||||
|
|
||||||
// The interval changes after the first report. The second report is sent
|
|
||||||
// before Target's first check since then, and the third after it. A fourth
|
|
||||||
// would hold no check the first three did not, so it is not sent.
|
|
||||||
test("checks recorded before an interval change and not yet reported go in the next report, with the status of the last check, and no check is reported twice", async (t) => {
|
|
||||||
const state = new AppState([
|
|
||||||
{ name: "Target", url: "https://target.test" },
|
|
||||||
]);
|
|
||||||
const target = state.local[0];
|
|
||||||
const reporter = new Reporter(state, "client", CONFIG.reportInterval);
|
|
||||||
t.mock.method(globalThis, "fetch", async () => ({ ok: true }));
|
|
||||||
|
|
||||||
target.pushSample(1000, { latency: 10, error: null });
|
|
||||||
await reporter.flush();
|
|
||||||
target.pushSample(2000, { latency: 20, error: null });
|
|
||||||
target.pushSample(3000, { latency: null, error: "timeout" });
|
|
||||||
target.reset();
|
|
||||||
await reporter.flush();
|
|
||||||
target.pushSample(4000, { latency: 40, error: null });
|
|
||||||
await reporter.flush();
|
|
||||||
await reporter.flush();
|
|
||||||
|
|
||||||
const name = "Target";
|
|
||||||
const url = "https://target.test";
|
|
||||||
assert.deepEqual(reportedHosts(), [
|
|
||||||
[
|
|
||||||
{
|
|
||||||
name,
|
|
||||||
url,
|
|
||||||
status: "online",
|
|
||||||
history: [{ t: 1000, latency: 10, error: null }],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[
|
|
||||||
{
|
|
||||||
name,
|
|
||||||
url,
|
|
||||||
status: "error",
|
|
||||||
history: [
|
|
||||||
{ t: 2000, latency: 20, error: null },
|
|
||||||
{ t: 3000, latency: null, error: "timeout" },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[
|
|
||||||
{
|
|
||||||
name,
|
|
||||||
url,
|
|
||||||
status: "online",
|
|
||||||
history: [{ t: 4000, latency: 40, error: null }],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
]);
|
|
||||||
// The page's history starts over.
|
|
||||||
assert.deepEqual(target.history, [
|
|
||||||
{ timestamp: 4000, latency: 40, error: null },
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
// The interval changes twice, after 60 checks each time, before the first
|
|
||||||
// report. Of the 120 checks the interval menu cleared, the report has the
|
|
||||||
// last 100 (CONFIG.maxHistoryPoints).
|
|
||||||
test("of the checks interval changes cleared before they were reported, the next report has the last 100", async (t) => {
|
|
||||||
const state = new AppState([
|
|
||||||
{ name: "Target", url: "https://target.test" },
|
|
||||||
]);
|
|
||||||
const target = state.local[0];
|
|
||||||
const reporter = new Reporter(state, "client", CONFIG.reportInterval);
|
|
||||||
t.mock.method(globalThis, "fetch", async () => ({ ok: true }));
|
|
||||||
|
|
||||||
for (let n = 1; n <= 120; n++) {
|
|
||||||
target.pushSample(n * 1000, { latency: 10, error: null });
|
|
||||||
if (n % 60 === 0) target.reset();
|
|
||||||
}
|
|
||||||
await reporter.flush();
|
|
||||||
|
|
||||||
const sent = reportedHosts()[0][0].history;
|
|
||||||
assert.equal(sent.length, 100);
|
|
||||||
assert.equal(sent[0].t, 21000);
|
|
||||||
assert.equal(sent.at(-1).t, 120000);
|
|
||||||
});
|
|
||||||
|
|
||||||
// The page shows < > " & and ' in a row's markup as
|
// The page shows < > " & and ' in a row's markup as
|
||||||
// < > " & and '.
|
// < > " & and '.
|
||||||
test(`a target whose name and URL hold < > " & and ' shows those characters in its row`, () => {
|
test(`a target whose name and URL hold < > " & and ' shows those characters in its row`, () => {
|
||||||
|
|||||||
+12
-38
@@ -257,14 +257,6 @@ function latencyStats(latencies) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// The status a check's error gives its host: "error" for a timeout,
|
|
||||||
// "offline" for any other error, and "online" for none.
|
|
||||||
function statusFromError(error) {
|
|
||||||
if (error === "timeout") return "error";
|
|
||||||
if (error) return "offline";
|
|
||||||
return "online";
|
|
||||||
}
|
|
||||||
|
|
||||||
export class HostState {
|
export class HostState {
|
||||||
constructor(host, pinned = false) {
|
constructor(host, pinned = false) {
|
||||||
this.name = host.name;
|
this.name = host.name;
|
||||||
@@ -273,10 +265,6 @@ export class HostState {
|
|||||||
// error }, or a round skipped while paused, { timestamp,
|
// error }, or a round skipped while paused, { timestamp,
|
||||||
// latency: null, paused: true }.
|
// latency: null, paused: true }.
|
||||||
this.history = [];
|
this.history = [];
|
||||||
// The last 100 (CONFIG.maxHistoryPoints) entries the interval menu
|
|
||||||
// has cleared from the history, which the reporter may not have
|
|
||||||
// sent yet.
|
|
||||||
this.clearedHistory = [];
|
|
||||||
this.lastLatency = null;
|
this.lastLatency = null;
|
||||||
this.status = "pending"; // 'online' | 'offline' | 'error' | 'pending'
|
this.status = "pending"; // 'online' | 'offline' | 'error' | 'pending'
|
||||||
this.pinned = pinned;
|
this.pinned = pinned;
|
||||||
@@ -292,7 +280,9 @@ export class HostState {
|
|||||||
});
|
});
|
||||||
this._trim();
|
this._trim();
|
||||||
this.lastLatency = result.latency;
|
this.lastLatency = result.latency;
|
||||||
this.status = statusFromError(result.error);
|
if (result.error === "timeout") this.status = "error";
|
||||||
|
else if (result.error) this.status = "offline";
|
||||||
|
else this.status = "online";
|
||||||
this.consecutiveFailures = result.error
|
this.consecutiveFailures = result.error
|
||||||
? this.consecutiveFailures + 1
|
? this.consecutiveFailures + 1
|
||||||
: 0;
|
: 0;
|
||||||
@@ -303,18 +293,6 @@ export class HostState {
|
|||||||
this._trim();
|
this._trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
// What the interval menu does to each host: the history starts over and
|
|
||||||
// the host waits for its next check. The history's entries move to
|
|
||||||
// clearedHistory, so the reporter still sends those it has not.
|
|
||||||
reset() {
|
|
||||||
this.clearedHistory = [...this.clearedHistory, ...this.history].slice(
|
|
||||||
-CONFIG.maxHistoryPoints,
|
|
||||||
);
|
|
||||||
this.history = [];
|
|
||||||
this.lastLatency = null;
|
|
||||||
this.status = "pending";
|
|
||||||
}
|
|
||||||
|
|
||||||
// The min, max, median and average latency of the checks in the history
|
// The min, max, median and average latency of the checks in the history
|
||||||
// that got an answer.
|
// that got an answer.
|
||||||
historyStats() {
|
historyStats() {
|
||||||
@@ -451,12 +429,9 @@ function getClientId() {
|
|||||||
|
|
||||||
// Build the delta report body the backend decodes, plus the new per-host
|
// Build the delta report body the backend decodes, plus the new per-host
|
||||||
// high-water marks. Pure function of the passed state: `hosts` is an array
|
// high-water marks. Pure function of the passed state: `hosts` is an array
|
||||||
// of { name, url, status, clearedHistory, history }, `since` maps a host url
|
// of { name, url, status, history }, `since` maps a host url to the Unix-ms
|
||||||
// to the Unix-ms timestamp of the last sample already reported for it, and
|
// timestamp of the last sample already reported for it, and `now` is a Date.
|
||||||
// `now` is a Date. Only non-paused samples newer than the mark are included,
|
// Only non-paused samples newer than the mark are included. Returns null
|
||||||
// from clearedHistory as well as history. A host whose status is pending,
|
|
||||||
// as the interval menu leaves it until its next check, is sent with the
|
|
||||||
// status of its last check, which is the last sample sent. Returns null
|
|
||||||
// when no host has an unreported sample.
|
// when no host has an unreported sample.
|
||||||
export function buildReport(hosts, clientId, now, since) {
|
export function buildReport(hosts, clientId, now, since) {
|
||||||
const reportHosts = [];
|
const reportHosts = [];
|
||||||
@@ -465,7 +440,7 @@ export function buildReport(hosts, clientId, now, since) {
|
|||||||
const mark = since.get(host.url) ?? 0;
|
const mark = since.get(host.url) ?? 0;
|
||||||
const samples = [];
|
const samples = [];
|
||||||
let high = mark;
|
let high = mark;
|
||||||
for (const p of [...host.clearedHistory, ...host.history]) {
|
for (const p of host.history) {
|
||||||
if (p.paused) continue;
|
if (p.paused) continue;
|
||||||
if (p.timestamp <= mark) continue;
|
if (p.timestamp <= mark) continue;
|
||||||
samples.push({
|
samples.push({
|
||||||
@@ -479,10 +454,7 @@ export function buildReport(hosts, clientId, now, since) {
|
|||||||
reportHosts.push({
|
reportHosts.push({
|
||||||
name: host.name,
|
name: host.name,
|
||||||
url: host.url,
|
url: host.url,
|
||||||
status:
|
status: host.status,
|
||||||
host.status === "pending"
|
|
||||||
? statusFromError(samples.at(-1).error)
|
|
||||||
: host.status,
|
|
||||||
history: samples,
|
history: samples,
|
||||||
});
|
});
|
||||||
marks.set(host.url, high);
|
marks.set(host.url, high);
|
||||||
@@ -509,7 +481,7 @@ export function buildReport(hosts, clientId, now, since) {
|
|||||||
// Only one report is ever in flight, and it is abandoned after half the
|
// Only one report is ever in flight, and it is abandoned after half the
|
||||||
// interval, so a slow POST can neither overlap the next report (which would
|
// interval, so a slow POST can neither overlap the next report (which would
|
||||||
// carry the same samples) nor stall reporting for good.
|
// carry the same samples) nor stall reporting for good.
|
||||||
export class Reporter {
|
class Reporter {
|
||||||
constructor(state, clientId, intervalMs) {
|
constructor(state, clientId, intervalMs) {
|
||||||
this.state = state;
|
this.state = state;
|
||||||
this.clientId = clientId;
|
this.clientId = clientId;
|
||||||
@@ -1530,7 +1502,9 @@ async function init() {
|
|||||||
|
|
||||||
// Clear all host history and reset state
|
// Clear all host history and reset state
|
||||||
for (const host of state.allHosts) {
|
for (const host of state.allHosts) {
|
||||||
host.reset();
|
host.history = [];
|
||||||
|
host.lastLatency = null;
|
||||||
|
host.status = "pending";
|
||||||
}
|
}
|
||||||
state.tickCount = 0;
|
state.tickCount = 0;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user