A network failure is printed and recorded with its reason (closes #196)

Node's `fetch` rejects every failure below HTTP as `fetch failed` and puts
the reason on `cause`. `describeError` in `src/errors.ts` gives the message
followed by the messages on the cause chain, and for an `AggregateError`,
which Node gives when every address of a host fails, the message of each
address. `run` prints it, and the backup logs and records it. `classify`
reads that text and calls a failure transient when `hasTransportCode`, now
exported by `src/retry.ts`, finds a transport errno on the chain. A failed
ML data fetch keeps its error as the cause. The bounded walk along the
chain moved from `src/retry.ts` to `src/errors.ts`.

A truncated download's message leaves the authentication failure to its
cause.

Model: opus-5-5

test/cli/backup-end-to-end.test.ts

@@ -329,6 +329,41 @@ const downloadOf = (input: RequestInfo | URL): number | undefined => {
         : undefined;
 };
 
+// The error Node gives for a connection to `address` that was refused.
+const refusedAt = (address: string): Error =>
+    Object.assign(new Error(`connect ECONNREFUSED ${address}`), {
+        code: "ECONNREFUSED",
+    });
+
+// What Node's `fetch` rejects with when the server refuses the connection, as
+// for every failure below HTTP: a TypeError that says only "fetch failed",
+// with the reason, and its errno, as its cause. When the server has several
+// addresses and each one refuses, the cause is an AggregateError with an empty
+// message, one error per address and the errno of the first. `reason` is the
+// error as quak prints it and records it in `failures.json`.
+const REFUSED = [
+    {
+        addresses: "its one address",
+        error: (): TypeError =>
+            new TypeError("fetch failed", { cause: refusedAt("1.2.3.4:443") }),
+        reason: "fetch failed: connect ECONNREFUSED 1.2.3.4:443",
+    },
+    {
+        addresses: "each of its two addresses",
+        error: (): TypeError =>
+            new TypeError("fetch failed", {
+                cause: Object.assign(
+                    new AggregateError([
+                        refusedAt("2001:db8::1:443"),
+                        refusedAt("1.2.3.4:443"),
+                    ]),
+                    { code: "ECONNREFUSED" },
+                ),
+            }),
+        reason: "fetch failed: connect ECONNREFUSED 2001:db8::1:443, connect ECONNREFUSED 1.2.3.4:443",
+    },
+];
+
 // The fake server, pushing onto `downloads` the ID of each file whose original
 // is downloaded.
 const recordingDownloads =
@@ -619,6 +654,115 @@ describe("quak backup", () => {
         expect(readdirSync(dir)).toEqual([]);
     });
 
+    // The server refuses every connection, at its one address or at each of
+    // several. quak tries each request again, here without waiting, and then
+    // the run stops at the refresh it starts with. The line it prints gives
+    // the reason Node's `fetch` put on the error's cause, so a cron log tells
+    // a refused connection from a name that did not resolve, and the exit
+    // code is 1.
+    it.each(REFUSED)(
+        "prints why the refresh could not reach the server at $addresses, and exits 1",
+        async ({ error, reason }) => {
+            const dir = mkdtempSync(join(root, "backup-"));
+            const fetch: typeof globalThis.fetch = async () => {
+                throw error();
+            };
+
+            const { code, stderr } = await backup(dir, {}, fetch);
+
+            expect(code).toBe(1);
+            expect(stderr).toBe(
+                "Starting backup...\n" +
+                    "Refreshing library...\n" +
+                    `quak: ${reason}\n`,
+            );
+        },
+    );
+
+    // The files CDN refuses every connection for toast.mp4, at its one
+    // address or at each of several. Once its retries are spent, the file
+    // goes into `failures.json` with the reason, as a transient failure,
+    // since a later run may well reach the CDN, and the lines the backup
+    // prints for it give the same reason. The rest of the account is backed
+    // up, and the exit code is 1.
+    it.each(REFUSED)(
+        "records why a download could not reach the server at $addresses in failures.json, as transient, and exits 1",
+        async ({ error, reason }) => {
+            let attempts = 0;
+            const fetch: typeof globalThis.fetch = async (input, init) => {
+                if (downloadOf(input) === 200) {
+                    attempts++;
+                    throw error();
+                }
+                return server(input, init);
+            };
+            const dir = mkdtempSync(join(root, "backup-"));
+
+            const { code, stderr } = await backup(dir, {}, fetch);
+
+            expect(code).toBe(1);
+            expect(attempts).toBe(UNATTENDED_RETRY_OPTIONS.attempts);
+            const failures = readJSON(join(dir, "failures.json"));
+            expect(Object.keys(failures.files)).toEqual(["200"]);
+            expect(failures.files["200"]).toMatchObject({
+                fileID: 200,
+                title: "toast.mp4",
+                classification: "transient",
+                attempts: 1,
+                error: reason,
+            });
+            expect(stderr).toContain(`FAILED original toast.mp4: ${reason}\n`);
+            expect(stderr).toContain(
+                `[Friend's Wedding] toast.mp4 (id 200): ${reason}\n`,
+            );
+            expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual(
+                Buffer.from(JPEG_WITH_EXIF),
+            );
+            expect(existsSync(join(dir, DAY, "2024-03-01.200.mp4"))).toBe(
+                false,
+            );
+        },
+    );
+
+    // The name of Ente's API server does not resolve when quak fetches the ML
+    // data, after every original has been backed up. No file has ML data
+    // cached yet, so each goes into `failures.json` with the reason, as a
+    // transient failure, as a download that could not reach the server does,
+    // and the exit code is 1.
+    it("records why the ML data fetch could not reach the server in failures.json, as transient, and exits 1", async () => {
+        const fetch: typeof globalThis.fetch = async (input, init) => {
+            if (new URL(String(input)).pathname === "/files/data/fetch") {
+                throw new TypeError("fetch failed", {
+                    cause: Object.assign(
+                        new Error("getaddrinfo ENOTFOUND api.ente.io"),
+                        { code: "ENOTFOUND" },
+                    ),
+                });
+            }
+            return server(input, init);
+        };
+        const dir = mkdtempSync(join(root, "backup-"));
+
+        const { code, stderr } = await backup(dir, {}, fetch);
+
+        expect(code).toBe(1);
+        const error =
+            "ML data: fetch failed: getaddrinfo ENOTFOUND api.ente.io";
+        expect(stderr).toContain(`FAILED ${error}\n`);
+        const failures = readJSON(join(dir, "failures.json"));
+        expect(
+            Object.values(failures.files).map((f) => [
+                f.fileID,
+                f.classification,
+                f.error,
+            ]),
+        ).toEqual([
+            [100, "transient", error],
+            [200, "transient", error],
+            [300, "transient", error],
+        ]);
+    });
+
     // Something changed beach.jpg on disk after it was backed up. A run with
     // --verify hashes every stored original, finds that one no longer matches
     // the hash Ente recorded, says so, and downloads it again; it put back the