A network failure is printed and recorded with its reason (closes #196)

Node's `fetch` rejects every failure below HTTP as `fetch failed` and puts
the reason on `cause`. `describeError` in `src/errors.ts` gives the message
followed by the messages on the cause chain, and for an `AggregateError`,
which Node gives when every address of a host fails, the message of each
address. `run` prints it, and the backup logs and records it. `classify`
reads that text and calls a failure transient when `hasTransportCode`, now
exported by `src/retry.ts`, finds a transport errno on the chain. A failed
ML data fetch keeps its error as the cause. The bounded walk along the
chain moved from `src/retry.ts` to `src/errors.ts`.

A truncated download's message leaves the authentication failure to its
cause.

Model: opus-5-5

src/backup.ts

@@ -82,6 +82,7 @@ import {
     init,
 } from "./crypto/index.js";
 import { removeLeftoverTempFiles } from "./download/index.js";
+import { describeError } from "./errors.js";
 import { sanitizeFileName, withExtension } from "./filename.js";
 import {
     copyAtomic,
@@ -94,6 +95,7 @@ import type { Undecryptable } from "./library/store.js";
 import { extractImageMetadata } from "./metadata-backup.js";
 import type { MLData } from "./mldata-fetch.js";
 import type { Collection, EnteFile, FileMetadata } from "./model/types.js";
+import { hasTransportCode } from "./retry.js";
 
 export type ProgressCallback = (message: string) => void;
 
@@ -219,11 +221,13 @@ const isPresent = (path: string): boolean => {
 
 // Best-effort classification for the ledger. Retryable server/network problems
 // are transient; refusals and local filesystem/decrypt errors are permanent;
-// anything else is unknown. Both the error code and message are inspected.
+// anything else is unknown. The error code and the error as `describeError`
+// gives it are inspected, and so is each errno on the error's cause chain,
+// where Node's `fetch` puts it.
 const classify = (err: unknown): FailureClass => {
+    if (hasTransportCode(err)) return "transient";
     const e = err as NodeJS.ErrnoException;
-    const text =
-        `${e?.code ?? ""} ${err instanceof Error ? err.message : String(err)}`.toLowerCase();
+    const text = `${e?.code ?? ""} ${describeError(err)}`.toLowerCase();
     if (
         /timeout|timed out|econnreset|econnrefused|econnaborted|network|socket|eai_again|throttl|temporarily|429|500|502|503|504/.test(
             text,
@@ -241,9 +245,6 @@ const classify = (err: unknown): FailureClass => {
     return "unknown";
 };
 
-const errorMessage = (err: unknown): string =>
-    err instanceof Error ? err.message : String(err);
-
 // Ensure `linkPath` is a symlink to `target`, rebuilding a missing, wrong, or
 // non-symlink entry. Throws on failure (a directory in the way, no permission)
 // so the caller records it and moves on rather than aborting the run.
@@ -470,7 +471,7 @@ const imageMetadataFor = async (
         const bytes = await readFile(originalPath);
         return { imageMetadata: extractImageMetadata(bytes) ?? {} };
     } catch (err) {
-        return { imageMetadataError: errorMessage(err) };
+        return { imageMetadataError: describeError(err) };
     }
 };
 
@@ -610,7 +611,7 @@ const runLockedBackup = async (
         // and thumbnail both fail this run must not double its attempt count
         // or appear twice in errors.
         if (failedThisRun.has(fileID)) return;
-        const error = errorMessage(err);
+        const error = describeError(err);
         errors.push({ fileID, title, collection, error });
         const prior = ledger.get(fileID);
         ledger.set(fileID, {
@@ -681,7 +682,7 @@ const runLockedBackup = async (
                 }
             } catch (err) {
                 log(
-                    `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
+                    `FAILED verifying original ${file.metadata.title}: ${describeError(err)}`,
                 );
                 recordFailure(
                     file.id,
@@ -717,7 +718,9 @@ const runLockedBackup = async (
             }
             downloaded++;
         } catch (err) {
-            log(`FAILED original ${file.metadata.title}: ${errorMessage(err)}`);
+            log(
+                `FAILED original ${file.metadata.title}: ${describeError(err)}`,
+            );
             recordFailure(
                 file.id,
                 file.metadata.title,
@@ -785,12 +788,14 @@ const runLockedBackup = async (
     // instead and is recorded as failed. The next run fetches its ML data
     // again because none is cached.
     if (includeOriginals) {
+        let mlDataFailure: unknown;
         let mlDataError: string | undefined;
         try {
             log("Fetching ML data...");
             await lib.fetchMLData();
         } catch (err) {
-            mlDataError = errorMessage(err);
+            mlDataFailure = err;
+            mlDataError = describeError(err);
             log(`FAILED ML data: ${mlDataError}`);
         }
         for (const file of distinct.values()) {
@@ -814,7 +819,7 @@ const runLockedBackup = async (
                     file.id,
                     file.metadata.title,
                     collectionName.get(file.collectionID) ?? "",
-                    new Error(`ML data: ${mlDataError}`),
+                    new Error("ML data", { cause: mlDataFailure }),
                 );
             } else {
                 writeSidecar(path, file, { mlData }, image);
@@ -846,7 +851,7 @@ const runLockedBackup = async (
             downloadDirectory,
         );
     } catch (err) {
-        log(`FAILED removing old album directories: ${errorMessage(err)}`);
+        log(`FAILED removing old album directories: ${describeError(err)}`);
     }
 
     for (const c of collections) {
@@ -868,7 +873,9 @@ const runLockedBackup = async (
         try {
             removeStaleLinks(colDir, new Set(linkNames), downloadDirectory);
         } catch (err) {
-            log(`FAILED removing old links in ${c.name}: ${errorMessage(err)}`);
+            log(
+                `FAILED removing old links in ${c.name}: ${describeError(err)}`,
+            );
         }
 
         const metaFiles = files.map((f) => ({
@@ -885,7 +892,7 @@ const runLockedBackup = async (
                 );
             } catch (err) {
                 log(
-                    `FAILED symlink ${c.name}/${linkName}: ${errorMessage(err)}`,
+                    `FAILED symlink ${c.name}/${linkName}: ${describeError(err)}`,
                 );
                 recordFailure(
                     link.file.id,

src/cli-run.ts

@@ -2,14 +2,17 @@
 
 import type { Writable } from "node:stream";
 import { ApiError } from "./api/client.js";
+import { describeError } from "./errors.js";
 
 // Run a command and exit with its code once stdout/stderr have drained.
 // Exiting before the drain can truncate piped output, and the library can keep
 // the event loop alive after a command returns, so a plain return could hang.
 // An error the command throws is printed as one `quak: MESSAGE` line, without
-// the stack trace, and exits 1. A 401 from the server means it no longer
-// accepts the saved session: that prints one line saying to log in again and
-// exits 3, as a missing or corrupt session file does.
+// the stack trace, and exits 1. MESSAGE is the error as `describeError` gives
+// it, so it carries the reason Node's `fetch` puts on `cause`. A 401 from the
+// server means it no longer accepts the saved session: that prints one line
+// saying to log in again and exits 3, as a missing or corrupt session file
+// does.
 export const run = async (
     command: Promise<number>,
     stdout: Writable,
@@ -26,9 +29,7 @@ export const run = async (
             );
             code = 3;
         } else {
-            stderr.write(
-                `quak: ${err instanceof Error ? err.message : String(err)}\n`,
-            );
+            stderr.write(`quak: ${describeError(err)}\n`);
             code = 1;
         }
     }

src/download/index.ts

@@ -135,15 +135,17 @@ const streamDecrypt = async (
                     // cannot tell a partial chunk from a corrupt one, so this
                     // is reported as the truncation it almost always is, with
                     // the authentication failure kept as the error's cause.
-                    // Only the pull is guarded: a sink failure on a chunk
-                    // that did authenticate is a disk error, not a
+                    // The message leaves that failure to the cause, which
+                    // `describeError` appends to what `quak backup` prints and
+                    // records. Only the pull is guarded: a sink failure on a
+                    // chunk that did authenticate is a disk error, not a
                     // truncation.
                     let pulled;
                     try {
                         pulled = pullStreamChunk(state, buffer);
                     } catch (err) {
                         throw new TruncatedStreamError(
-                            `download: stream truncated: response body ended with ${buffer.length} trailing bytes that did not authenticate as a final chunk (transfer stopped mid-chunk, or the data is corrupt)`,
+                            `download: stream truncated: response body ended with ${buffer.length} trailing bytes (transfer stopped mid-chunk, or the data is corrupt)`,
                             { cause: err },
                         );
                     }

src/errors.ts

@@ -1,10 +1,13 @@
-// Error types that more than one layer of quak needs to recognise.
+// Error types that more than one layer of quak needs to recognise, and how an
+// error is described in what quak prints and in `failures.json`.
 //
-// They live here rather than beside the code that throws them so that the
+// The types live here rather than beside the code that throws them so that the
 // retry classifier can identify them without importing the HTTP client or the
 // download layer — both of which import the classifier. `ApiError` is
 // re-exported from `src/api/client.ts`, which is where callers have always
-// imported it from and where it still belongs conceptually.
+// imported it from and where it still belongs conceptually. The walk along an
+// error's `cause` chain is here too, because the retry classifier reads the
+// chain for errnos and `describeError` for messages.
 
 export class ApiError extends Error {
     readonly status: number;
@@ -43,3 +46,68 @@ export class TruncatedStreamError extends Error {
         this.name = "TruncatedStreamError";
     }
 }
+
+// `cause` is an arbitrary user-settable property and nothing prevents it from
+// forming a cycle, so the walk is bounded. Hanging the process would be worse
+// than misclassifying an error or describing only part of it.
+const MAX_CAUSE_DEPTH = 8;
+
+// The error and each error on its `cause` chain, top first, at most
+// `MAX_CAUSE_DEPTH` of them. `complete` is false when the walk stopped at the
+// depth limit with more of the chain still below it.
+export const causeChain = (
+    err: unknown,
+): { chain: object[]; complete: boolean } => {
+    const chain: object[] = [];
+    let current: unknown = err;
+    for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {
+        if (current === null || typeof current !== "object") {
+            return { chain, complete: true };
+        }
+        chain.push(current);
+        const { cause } = current as { cause?: unknown };
+        if (cause === current) return { chain, complete: true };
+        current = cause;
+    }
+    return { chain, complete: current === null || typeof current !== "object" };
+};
+
+// An error as quak prints it and records it in `failures.json`: its message,
+// then the message of each error on its `cause` chain, joined with ": ". Node's
+// `fetch` rejects with just "fetch failed" and puts the reason on `cause`, so
+// this gives "fetch failed: connect ECONNREFUSED 1.2.3.4:443". When the host
+// has several addresses and every one fails, the cause is an `AggregateError`
+// with an empty message and one error per address in `errors`, so an
+// `AggregateError` is given as the messages of its errors, joined with ", ".
+// A cause that is not an object, such as a thrown string, is given as
+// `String(cause)`. Each message is trimmed of the white space around it, as
+// OpenSSL ends its own with a newline. A message the text already holds is
+// left out.
+export const describeError = (err: unknown): string => {
+    if (!(err instanceof Error)) return String(err);
+    const { chain } = causeChain(err);
+    const messages = chain
+        .slice(1)
+        .map((cause) =>
+            cause instanceof AggregateError
+                ? cause.errors
+                      .map((e: unknown) =>
+                          (e instanceof Error ? e.message : String(e)).trim(),
+                      )
+                      .join(", ")
+                : (cause as { message?: unknown }).message,
+        );
+    // `causeChain` stops at a cause that is not an object, so that one is read
+    // from the last error on the chain.
+    const last = chain[chain.length - 1] as { cause?: unknown };
+    if (last.cause !== undefined && typeof last.cause !== "object") {
+        messages.push(String(last.cause));
+    }
+    let text = err.message.trim();
+    for (const message of messages) {
+        if (typeof message !== "string") continue;
+        const trimmed = message.trim();
+        if (!text.includes(trimmed)) text += `: ${trimmed}`;
+    }
+    return text;
+};

src/retry.ts

@@ -10,7 +10,7 @@
 // declining to retry a transient one costs a single file that the next run
 // picks up anyway. When the evidence is ambiguous, fail fast.
 
-import { ApiError, TruncatedStreamError } from "./errors.js";
+import { ApiError, causeChain, TruncatedStreamError } from "./errors.js";
 
 export interface RetryOptions {
     // Total calls, not retries: `attempts: 1` disables retrying.
@@ -89,11 +89,6 @@ const TRANSPORT_CODES = new Set([
 // narrowed. See `isSafeToReplay`.
 const CONNECT_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "ECONNREFUSED"]);
 
-// `cause` is an arbitrary user-settable property and nothing prevents it from
-// forming a cycle, so the walk is bounded. Hanging the process would be a
-// worse outcome than any misclassification.
-const MAX_CAUSE_DEPTH = 8;
-
 // Collect every `code` in an error's cause chain. undici does not put the
 // errno on the error it throws — it hangs the underlying socket error off
 // `cause`, sometimes more than one level down — so a classifier that only read
@@ -101,20 +96,21 @@ const MAX_CAUSE_DEPTH = 8;
 // connection permanent. `complete` is false when the walk stopped at the
 // depth limit with more of the chain still below it.
 const causeCodes = (err: unknown): { codes: string[]; complete: boolean } => {
+    const { chain, complete } = causeChain(err);
     const codes: string[] = [];
-    let current: unknown = err;
-    for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {
-        if (current === null || typeof current !== "object") {
-            return { codes, complete: true };
-        }
-        const { code, cause } = current as { code?: unknown; cause?: unknown };
+    for (const link of chain) {
+        const { code } = link as { code?: unknown };
         if (typeof code === "string") codes.push(code);
-        if (cause === current) return { codes, complete: true };
-        current = cause;
     }
-    return { codes, complete: current === null || typeof current !== "object" };
+    return { codes, complete };
 };
 
+// Does the error's cause chain hold the errno of a transport failure, one of
+// `TRANSPORT_CODES`? `isRetryable` retries such an error, and `classify` in
+// `src/backup.ts` calls it transient.
+export const hasTransportCode = (err: unknown): boolean =>
+    causeCodes(err).codes.some((code) => TRANSPORT_CODES.has(code));
+
 const isAbort = (err: unknown): boolean => {
     if (err === null || typeof err !== "object") return false;
     const { name } = err as { name?: unknown };
@@ -158,7 +154,7 @@ export const isRetryable = (err: unknown): boolean => {
     // have succeeded; the cost of the imprecision is bounded by the attempt
     // count.
     if (err instanceof TypeError) return true;
-    return causeCodes(err).codes.some((code) => TRANSPORT_CODES.has(code));
+    return hasTransportCode(err);
 };
 
 // Could the first attempt already have taken effect on the server?