The binary make install builds retries a refused or unresolvable connection (closes #206)

Bun's `fetch`, which `~/bin/quak` runs under, rejects with a plain `Error`
that has no `cause` and a code of bun's own. Running the built binary against
local listeners gave `ConnectionRefused` for a refused connection and a failed
TLS handshake, and `ConnectionRefused` or `FailedToOpenSocket` for a name that
does not resolve. Both join `TRANSPORT_CODES` in `src/retry.ts`, so they are
retried and called transient. They stay out of `CONNECT_CODES`, so a `POST` or
`PUT` is not replayed on them, because bun also gives them when it resends a
request on a new connection. `classify` in `src/backup.ts` records a
`TimeoutError` as transient, so a stalled download is no longer `unknown`.

Not changed: a certificate bun rejects is not retried.

Model: opus-5-5

test/cli/backup-end-to-end.test.ts

@@ -275,13 +275,15 @@ afterAll(() => {
 // Run `quak backup <dir>` as `bin/quak.ts` runs it, sending its requests
 // through `fetch`, and return its exit code and everything it printed on
 // stderr. Its requests are retried as those of `quak backup` are, except that
-// a retry does not wait. Its cache is beside `dir`, so each run of one
-// directory finds the cache the run before it left, as cron's runs find the
-// user's.
+// a retry does not wait. A download is given up after `downloadTimeoutMs` with
+// no bytes received, 60 seconds unless given. Its cache is beside `dir`, so
+// each run of one directory finds the cache the run before it left, as cron's
+// runs find the user's.
 const backup = async (
     dir: string,
     opts: { verify?: boolean } = {},
     fetch = server,
+    downloadTimeoutMs?: number,
 ): Promise<{ code: number; stderr: string }> => {
     let printed = "";
     const stderr = new Writable({
@@ -308,6 +310,7 @@ const backup = async (
             loadSession(path, {
                 fetch,
                 retry: { ...UNATTENDED_RETRY_OPTIONS, sleep: async () => {} },
+                downloadTimeoutMs,
             }),
         login: prompted,
         prompt: prompted,
@@ -724,6 +727,81 @@ describe("quak backup", () => {
         },
     );
 
+    // The download of toast.mp4 fails as bun's `fetch` reports a failure, in
+    // the binary `make install` builds: a plain `Error` with bun's own code
+    // and message, and no cause. These are what bun gave for a refused
+    // connection, a name that did not resolve and a reset connection. The
+    // file is tried as often as under Node, and goes into `failures.json`
+    // with that message, as a transient failure.
+    it.each([
+        [
+            "ConnectionRefused",
+            "Unable to connect. Is the computer able to access the url?",
+        ],
+        ["FailedToOpenSocket", "Was there a typo in the url or port?"],
+        [
+            "ECONNRESET",
+            "The socket connection was closed unexpectedly. For more information, pass `verbose: true` in the second argument to fetch()",
+        ],
+    ])(
+        "records a download that failed with bun's %s in failures.json, as transient",
+        async (bunCode, message) => {
+            let attempts = 0;
+            const fetch: typeof globalThis.fetch = async (input, init) => {
+                if (downloadOf(input) === 200) {
+                    attempts++;
+                    throw Object.assign(new Error(message), {
+                        code: bunCode,
+                        errno: 0,
+                    });
+                }
+                return server(input, init);
+            };
+            const dir = mkdtempSync(join(root, "backup-"));
+
+            const { code } = await backup(dir, {}, fetch);
+
+            expect(code).toBe(1);
+            expect(attempts).toBe(UNATTENDED_RETRY_OPTIONS.attempts);
+            const failures = readJSON(join(dir, "failures.json"));
+            expect(failures.files["200"]).toMatchObject({
+                classification: "transient",
+                error: message,
+            });
+        },
+    );
+
+    // The download of toast.mp4 stalls: the server sends the headers and then
+    // never a byte of the body. Each attempt is given up after 50 ms, quak's
+    // own deadline for this run, with a `TimeoutError` whose message says
+    // only that the download stalled. The file goes into `failures.json` as a
+    // transient failure, as a request that ran out of time is retried.
+    it("records a stalled download in failures.json, as transient", async () => {
+        let attempts = 0;
+        const fetch: typeof globalThis.fetch = async (input, init) => {
+            if (downloadOf(input) === 200) {
+                attempts++;
+                return new Response(
+                    new ReadableStream<Uint8Array>({
+                        pull: () => new Promise<void>(() => {}),
+                    }),
+                );
+            }
+            return server(input, init);
+        };
+        const dir = mkdtempSync(join(root, "backup-"));
+
+        const { code } = await backup(dir, {}, fetch, 50);
+
+        expect(code).toBe(1);
+        expect(attempts).toBe(UNATTENDED_RETRY_OPTIONS.attempts);
+        const failures = readJSON(join(dir, "failures.json"));
+        expect(failures.files["200"]).toMatchObject({
+            classification: "transient",
+            error: "download stalled: no bytes received for 50 ms",
+        });
+    });
+
     // The name of Ente's API server does not resolve when quak fetches the ML
     // data, after every original has been backed up. No file has ML data
     // cached yet, so each goes into `failures.json` with the reason, as a

test/retry/retry.test.ts

@@ -77,6 +77,14 @@ const recordingSleep = (): {
 const errnoError = (code: string, message = code): Error =>
     Object.assign(new Error(message), { code });
 
+/**
+ * An error shaped like what bun's `fetch` rejects with for a failure below
+ * HTTP: a plain `Error` with bun's own message, a `code`, an `errno` of 0, and
+ * no `cause`.
+ */
+const bunError = (code: string, message: string): Error =>
+    Object.assign(new Error(message), { code, errno: 0 });
+
 // ---------------------------------------------------------------------------
 // Classification
 // ---------------------------------------------------------------------------
@@ -244,6 +252,82 @@ describe("isRetryable: transport failures", () => {
     });
 });
 
+describe("isRetryable and isSafeToReplay: bun's fetch errors", () => {
+    // The binary `make install` builds runs quak under bun, whose `fetch`
+    // does not reject as Node's does. Each shape here was found by running
+    // that binary, built with bun 1.3.13, against listeners on 127.0.0.1 and
+    // a name under `.invalid`.
+
+    it("retries a refused connection, and does not replay it", () => {
+        // Bun gives the same error when the TLS handshake fails, and when a
+        // new connection it opened to resend a request is refused. That
+        // request may already have reached the server on a connection bun
+        // kept open from an earlier request.
+        const refused = bunError(
+            "ConnectionRefused",
+            "Unable to connect. Is the computer able to access the url?",
+        );
+        expect(isRetryable(refused)).toBe(true);
+        expect(isSafeToReplay(refused)).toBe(false);
+    });
+
+    it("retries a name that does not resolve, with either code bun gives, and does not replay it", () => {
+        // For the same name bun gave `ConnectionRefused` on some requests and
+        // `FailedToOpenSocket` on others. Either can also come from a new
+        // connection bun opened to resend a request, as above.
+        for (const unresolved of [
+            bunError(
+                "ConnectionRefused",
+                "Unable to connect. Is the computer able to access the url?",
+            ),
+            bunError(
+                "FailedToOpenSocket",
+                "Was there a typo in the url or port?",
+            ),
+        ]) {
+            expect(isRetryable(unresolved)).toBe(true);
+            expect(isSafeToReplay(unresolved)).toBe(false);
+        }
+    });
+
+    it("retries a connection closed before the response ended, and does not replay it", () => {
+        // Bun gives Node's errno, `ECONNRESET`, for a reset and for a close,
+        // before the response or part-way through the body. The request may
+        // already have reached the server.
+        const reset = bunError(
+            "ECONNRESET",
+            "The socket connection was closed unexpectedly. For more information, pass `verbose: true` in the second argument to fetch()",
+        );
+        expect(isRetryable(reset)).toBe(true);
+        expect(isSafeToReplay(reset)).toBe(false);
+    });
+
+    it("retries a request that ran out of time, and does not replay it", () => {
+        // What bun's fetch rejects with when `AbortSignal.timeout()`, which
+        // bounds each JSON request, fires.
+        const timedOut = new DOMException(
+            "The operation timed out.",
+            "TimeoutError",
+        );
+        expect(isRetryable(timedOut)).toBe(true);
+        expect(isSafeToReplay(timedOut)).toBe(false);
+    });
+
+    it("does not retry a certificate bun rejects", () => {
+        // A certificate that fails verification fails the same way however
+        // often it is asked. Under Node it is retried only because every
+        // `TypeError` is.
+        expect(
+            isRetryable(
+                bunError(
+                    "DEPTH_ZERO_SELF_SIGNED_CERT",
+                    "self signed certificate",
+                ),
+            ),
+        ).toBe(false);
+    });
+});
+
 describe("isRetryable: stream truncation versus corruption", () => {
     it("retries a truncated stream", () => {
         // Truncation is a transfer that stopped early. The bytes that did