README.md: rework against the review findings

The check sentence names HEAD for the S3 regions up front. The rate-limit
sentence states the safe rate plainly: an address that sends at most half
the limit in any 60 seconds is never refused. The upaas `TRUSTED_PROXIES`
entry again warns that a client connecting from a trusted address can
write its own `X-Forwarded-For`, that a connection through a published
Docker port can arrive from the Docker network's gateway, which the
default includes, and that set and empty, every client behind the proxy
shares one allowance. `TODO.md` gets the Completed Steps entry for
#189, now that
#196 has merged. For
#195.

Model: fable-5-1

configs/security-headers.conf

@@ -8,10 +8,9 @@ add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" alway
 # Scripts and styles load only from the page's own origin. Inline ones
 # are blocked, style attributes in markup included, so style elements
 # through classes or element.style. data: images are for the favicon
-# in web/index.html. connect-src is * because the browser checks each
-# probe in web/src/main.js against it, and also every redirect the probe
-# follows, and several of those hosts redirect to others; a list of
-# hosts here would block those probes. It also covers the reports the
+# in web/index.html. connect-src is * because the page checks every
+# target in web/src/main.js. A list of hosts here would have to change
+# with every change to those targets. It also covers the reports the
 # page sends to its own origin.
 add_header Content-Security-Policy "default-src 'self'; connect-src *; img-src 'self' data:; object-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" always;
 

test/unit/main.test.js

@@ -180,6 +180,91 @@ test("a check fetches the target's URL as written, with no query string added",
     );
 });
 
+test("a check sends no Referer header, wherever the page is served from", async (t) => {
+    mockTargets(t, () => 10);
+    const check = measureLatency("https://fsn1-speed.hetzner.com");
+    t.mock.timers.tick(10);
+    assert.notEqual(await settled(check), "still waiting");
+    assert.equal(
+        fetch.mock.calls[0].arguments[1].referrerPolicy,
+        "no-referrer",
+    );
+});
+
+// These targets' home pages answer with a redirect. A check follows it and
+// times the request to the new URL too, often on another host. The Google
+// Cloud Console redirects only requests that do not say they come from
+// another site, such as curl's. Each target is checked at /robots.txt on
+// its own host, which answers without a redirect. An S3 region answers it
+// with 403, and any answer counts as reachable.
+test("a target whose home page redirects is checked at /robots.txt on its own host", () => {
+    const checked = new Map(
+        new AppState([]).wan.map((host) => [host.name, host.url]),
+    );
+    for (const [name, url] of [
+        ["AWS Console", "https://console.aws.amazon.com/robots.txt"],
+        ["Google Cloud Console", "https://console.cloud.google.com/robots.txt"],
+        ["Akamai", "https://www.akamai.com/robots.txt"],
+        ["B2", "https://api.backblazeb2.com/robots.txt"],
+        [
+            "S3 af-south-1 (Cape Town)",
+            "https://s3.af-south-1.amazonaws.com/robots.txt",
+        ],
+        [
+            "S3 eu-west-2 (London)",
+            "https://s3.eu-west-2.amazonaws.com/robots.txt",
+        ],
+        [
+            "S3 me-south-1 (Bahrain)",
+            "https://s3.me-south-1.amazonaws.com/robots.txt",
+        ],
+        [
+            "S3 ap-northeast-1 (Tokyo)",
+            "https://s3.ap-northeast-1.amazonaws.com/robots.txt",
+        ],
+        [
+            "S3 ap-southeast-1 (Singapore)",
+            "https://s3.ap-southeast-1.amazonaws.com/robots.txt",
+        ],
+        [
+            "S3 ap-southeast-2 (Sydney)",
+            "https://s3.ap-southeast-2.amazonaws.com/robots.txt",
+        ],
+        [
+            "S3 us-west-2 (Oregon)",
+            "https://s3.us-west-2.amazonaws.com/robots.txt",
+        ],
+        [
+            "S3 sa-east-1 (São Paulo)",
+            "https://s3.sa-east-1.amazonaws.com/robots.txt",
+        ],
+    ]) {
+        assert.equal(checked.get(name), url);
+    }
+});
+
+// Chrome keeps the connection to an S3 region open after a HEAD check, but
+// closes it after most GET checks. The Hetzner servers do not answer HEAD.
+test("a round checks each S3 region with a HEAD request and every other target with a GET request", async (t) => {
+    const state = new AppState([
+        { name: "Local CPE", url: "http://192.168.100.1" },
+    ]);
+    mockTargets(t, () => 10);
+    const round = tick(state, new AbortController().signal);
+    t.mock.timers.tick(10);
+    assert.notEqual(await settled(round), "still waiting");
+    const methods = new Map(
+        fetch.mock.calls.map((call) => [
+            call.arguments[0],
+            call.arguments[1].method,
+        ]),
+    );
+    for (const host of state.allHosts) {
+        const method = host.name.startsWith("S3 ") ? "HEAD" : "GET";
+        assert.equal(methods.get(host.url), method, host.name);
+    }
+});
+
 test("at a 30000ms interval, a target answering after 1000ms shows in its row while another target's check is still waiting", async (t) => {
     CONFIG.updateInterval = 30000;
     const state = new AppState([

web/src/main.js

@@ -67,51 +67,66 @@ export const CONFIG = {
 
 // WAN endpoints to monitor. These are used for the aggregate health/stats
 // display (min/max/avg). Ordered: personal, LLM APIs, big-3 cloud, then
-// CDN/hosting/other.
+// CDN/hosting/other. A check follows a redirect and times the request to
+// the new URL too. So a target whose home page redirects is checked at
+// /robots.txt on its own host. That URL answers without a redirect.
 const WAN_HOSTS = [
     { name: "datavi.be", url: "https://datavi.be" },
     { name: "Anthropic API", url: "https://api.anthropic.com" },
     { name: "OpenAI API", url: "https://api.openai.com" },
-    { name: "AWS Console", url: "https://console.aws.amazon.com" },
-    { name: "Google Cloud Console", url: "https://console.cloud.google.com" },
+    { name: "AWS Console", url: "https://console.aws.amazon.com/robots.txt" },
+    {
+        name: "Google Cloud Console",
+        url: "https://console.cloud.google.com/robots.txt",
+    },
     { name: "Microsoft Azure", url: "https://portal.azure.com" },
     { name: "Cloudflare", url: "https://www.cloudflare.com" },
     { name: "Fastly CDN", url: "https://www.fastly.com" },
-    { name: "Akamai", url: "https://www.akamai.com" },
+    { name: "Akamai", url: "https://www.akamai.com/robots.txt" },
     { name: "Google", url: "https://www.google.com" },
     { name: "GitHub", url: "https://github.com" },
-    { name: "B2", url: "https://api.backblazeb2.com" },
+    { name: "B2", url: "https://api.backblazeb2.com/robots.txt" },
+    // Chrome keeps the connection to an S3 region open after a HEAD check,
+    // but closes it after most GET checks.
     {
         name: "S3 af-south-1 (Cape Town)",
-        url: "https://s3.af-south-1.amazonaws.com",
+        url: "https://s3.af-south-1.amazonaws.com/robots.txt",
+        method: "HEAD",
     },
     {
         name: "S3 eu-west-2 (London)",
-        url: "https://s3.eu-west-2.amazonaws.com",
+        url: "https://s3.eu-west-2.amazonaws.com/robots.txt",
+        method: "HEAD",
     },
     {
         name: "S3 me-south-1 (Bahrain)",
-        url: "https://s3.me-south-1.amazonaws.com",
+        url: "https://s3.me-south-1.amazonaws.com/robots.txt",
+        method: "HEAD",
     },
     {
         name: "S3 ap-northeast-1 (Tokyo)",
-        url: "https://s3.ap-northeast-1.amazonaws.com",
+        url: "https://s3.ap-northeast-1.amazonaws.com/robots.txt",
+        method: "HEAD",
     },
     {
         name: "S3 ap-southeast-1 (Singapore)",
-        url: "https://s3.ap-southeast-1.amazonaws.com",
+        url: "https://s3.ap-southeast-1.amazonaws.com/robots.txt",
+        method: "HEAD",
     },
     {
         name: "S3 ap-southeast-2 (Sydney)",
-        url: "https://s3.ap-southeast-2.amazonaws.com",
+        url: "https://s3.ap-southeast-2.amazonaws.com/robots.txt",
+        method: "HEAD",
     },
     {
         name: "S3 us-west-2 (Oregon)",
-        url: "https://s3.us-west-2.amazonaws.com",
+        url: "https://s3.us-west-2.amazonaws.com/robots.txt",
+        method: "HEAD",
     },
     {
         name: "S3 sa-east-1 (São Paulo)",
-        url: "https://s3.sa-east-1.amazonaws.com",
+        url: "https://s3.sa-east-1.amazonaws.com/robots.txt",
+        method: "HEAD",
     },
     // Hetzner regional speed test servers — genuine per-DC endpoints
     {
@@ -271,6 +286,7 @@ export class HostState {
     constructor(host, pinned = false) {
         this.name = host.name;
         this.url = host.url;
+        this.method = host.method;
         // Each entry is either a check's result, { timestamp, latency,
         // error }, or a round skipped while paused, { timestamp,
         // latency: null, paused: true }.
@@ -567,16 +583,18 @@ export class Reporter {
 
 // --- Latency Measurement -----------------------------------------------------
 
-// Checks one target. The check times out after CONFIG.requestTimeout; the
-// caller can give it up sooner through the optional signal, which also ends
-// it as a timeout. A failed check's reason says what went wrong and after
-// how long; a check that answered has none.
+// Checks one target with the given HTTP method, GET when none is given. The
+// check times out after CONFIG.requestTimeout; the caller can give it up
+// sooner through the optional signal, which also ends it as a timeout. A
+// failed check's reason says what went wrong and after how long; a check
+// that answered has none.
 //
-// The URL is fetched as written, with nothing added to it: the Hetzner
-// speed-test servers close the connection without an answer when the URL
-// has a query string, and cache: "no-store" keeps the browser's cache out
-// of the measurement.
-export async function measureLatency(url, signal) {
+// The Hetzner speed-test servers close the connection without an answer
+// when the URL has a query string or the request has a Referer header. So
+// the URL is fetched as written, and referrerPolicy: "no-referrer" sends
+// no Referer header, wherever the page is served from. cache: "no-store"
+// keeps the browser's cache out of the measurement.
+export async function measureLatency(url, signal, method = "GET") {
     const controller = new AbortController();
     const timeoutId = setTimeout(
         () => controller.abort(),
@@ -588,9 +606,10 @@ export async function measureLatency(url, signal) {
 
     try {
         await fetch(url, {
-            method: "GET",
+            method,
             mode: "no-cors",
             cache: "no-store",
+            referrerPolicy: "no-referrer",
             signal: controller.signal,
         });
         const latency = Math.round(performance.now() - start);
@@ -1290,7 +1309,7 @@ export async function tick(state, signal, onOffline) {
     // the round started.
     await Promise.all(
         state.allHosts.map(async (host) => {
-            const r = await measureLatency(host.url, signal);
+            const r = await measureLatency(host.url, signal, host.method);
             if (state.paused || signal.aborted || state.tickCount === 0) {
                 return;
             }
@@ -1363,7 +1382,7 @@ function startRecoveryProbe(state, startRounds) {
         const checks = new AbortController();
         state._recoveryProbeChecks = checks;
         for (const host of canaries) {
-            measureLatency(host.url, checks.signal).then((r) => {
+            measureLatency(host.url, checks.signal, host.method).then((r) => {
                 if (r.error !== null || checks.signal.aborted) return;
                 log.notice("Recovery probe: connectivity detected");
                 stopRecoveryProbe(state);