From e93bb06330dfae9fdeac202c1f1460ab6837a296 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 15 Oct 2026 23:13:31 +0000 Subject: [PATCH] A network failure is printed and recorded with its reason (closes #196) Node's `fetch` rejects every failure below HTTP as `fetch failed` and puts the reason on `cause`, so quak printed and recorded only that, and `failures.json` called the failure `unknown`. `describeError` in `src/errors.ts` gives the message followed by the messages on the cause chain; `run` prints it, and the backup logs and records it. `classify` calls a failure transient when `hasTransportCode`, now exported by `src/retry.ts`, finds a transport errno on the chain. The bounded walk along the chain moved from `src/retry.ts` to `src/errors.ts`, so both use one walk. Every message the backup logs or records, `imageMetadataError` and `mlDataError` included, now carries its causes too. Model: opus-5-5 --- README.md | 10 ++++- TODO.md | 19 ++++++-- src/backup.ts | 9 ++-- src/cli-run.ts | 13 +++--- src/errors.ts | 51 +++++++++++++++++++-- src/retry.ts | 28 +++++------- test/cli/backup-end-to-end.test.ts | 71 ++++++++++++++++++++++++++++++ test/cli/run.test.ts | 29 ++++++++++++ 8 files changed, 198 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 12fd670..b235f84 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,13 @@ matters. | `2` | Another backup of the same directory is running. This run sent no request and changed nothing. | | `3` | There is no usable session: none is saved, the saved one is corrupt, or the server no longer accepts it. Run `quak login` as the user the cron job runs as. | +`` is the error's message, followed by the message of each error that +caused it. Node's `fetch` reports every failure below HTTP as just +`fetch failed` and gives the reason as its cause, so a run that could not reach +the server prints, for example, +`quak: fetch failed: connect ECONNREFUSED 1.2.3.4:443`. A file whose download +failed that way is in `failures.json` with that error, as a transient failure. + After a `1`, the next run fetches each missing original again, fetches the ML data that is not cached, and rebuilds the album links. An album quak could not decrypt, or a file in `failures.json` because quak could not decrypt it, keeps @@ -347,7 +354,8 @@ quak/ exif.ts EXIF read from an image's bytes with exifreader mldata-fetch.ts fetch + decrypt per-file ML data filename.ts safe file names from server metadata - errors.ts error types shared across layers + errors.ts error types shared across layers, and how an error + is described retry.ts retry classifier + exponential backoff with jitter thumbnails.ts detect + regenerate missing thumbnails client.ts high-level Client class assembled from the above diff --git a/TODO.md b/TODO.md index 1aa7f9b..61d33ea 100644 --- a/TODO.md +++ b/TODO.md @@ -17,15 +17,28 @@ pre-1.0 # Next Step -None: no implementation work is open. The check of quak against a live Ente -account, https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak -has, so it waits on him. +The backup issues https://git.eeqj.de/sneak/quak/issues/197, +https://git.eeqj.de/sneak/quak/issues/198 and +https://git.eeqj.de/sneak/quak/issues/199, and the README issue +https://git.eeqj.de/sneak/quak/issues/200, are open. The check of quak against a +live Ente account, https://git.eeqj.de/sneak/quak/issues/187, needs an account +only sneak has, so it waits on him. Tagging and releases are decided by sneak alone, and happen only when he declares one. # Completed Steps +- 2026-10-15: A network failure is printed and recorded with its reason, not as + just `fetch failed` (issue 196). `describeError` in `src/errors.ts` gives an + error's message followed by the message of each error on its `cause` chain, + such as `fetch failed: connect ECONNREFUSED 1.2.3.4:443`. `run` prints that, + and the backup logs it and records it in `failures.json`, where `classify` now + calls the failure transient when the chain holds an errno that + `hasTransportCode` in `src/retry.ts` recognises. The walk along the chain + moved from `src/retry.ts` to `src/errors.ts`, which both use. Tests refuse + every connection of a backup, and then only one file's download. + - 2026-10-14: `lib.backup()`, and so `quak backup`, works on up to the content pool's limit of originals at once, 5 by default, instead of one at a time (issue 182). That many workers take files from one shared list, each doing all diff --git a/src/backup.ts b/src/backup.ts index 488ff30..48744bd 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -82,6 +82,7 @@ import { init, } from "./crypto/index.js"; import { removeLeftoverTempFiles } from "./download/index.js"; +import { describeError } from "./errors.js"; import { sanitizeFileName, withExtension } from "./filename.js"; import { copyAtomic, @@ -94,6 +95,7 @@ import type { Undecryptable } from "./library/store.js"; import { extractImageMetadata } from "./metadata-backup.js"; import type { MLData } from "./mldata-fetch.js"; import type { Collection, EnteFile, FileMetadata } from "./model/types.js"; +import { hasTransportCode } from "./retry.js"; export type ProgressCallback = (message: string) => void; @@ -219,8 +221,10 @@ const isPresent = (path: string): boolean => { // Best-effort classification for the ledger. Retryable server/network problems // are transient; refusals and local filesystem/decrypt errors are permanent; -// anything else is unknown. Both the error code and message are inspected. +// anything else is unknown. The error code and message are inspected, and so +// is each errno on the error's cause chain, where Node's `fetch` puts it. const classify = (err: unknown): FailureClass => { + if (hasTransportCode(err)) return "transient"; const e = err as NodeJS.ErrnoException; const text = `${e?.code ?? ""} ${err instanceof Error ? err.message : String(err)}`.toLowerCase(); @@ -241,8 +245,7 @@ const classify = (err: unknown): FailureClass => { return "unknown"; }; -const errorMessage = (err: unknown): string => - err instanceof Error ? err.message : String(err); +const errorMessage = (err: unknown): string => describeError(err); // Ensure `linkPath` is a symlink to `target`, rebuilding a missing, wrong, or // non-symlink entry. Throws on failure (a directory in the way, no permission) diff --git a/src/cli-run.ts b/src/cli-run.ts index 027bc40..50bf822 100644 --- a/src/cli-run.ts +++ b/src/cli-run.ts @@ -2,14 +2,17 @@ import type { Writable } from "node:stream"; import { ApiError } from "./api/client.js"; +import { describeError } from "./errors.js"; // Run a command and exit with its code once stdout/stderr have drained. // Exiting before the drain can truncate piped output, and the library can keep // the event loop alive after a command returns, so a plain return could hang. // An error the command throws is printed as one `quak: MESSAGE` line, without -// the stack trace, and exits 1. A 401 from the server means it no longer -// accepts the saved session: that prints one line saying to log in again and -// exits 3, as a missing or corrupt session file does. +// the stack trace, and exits 1. MESSAGE is the error as `describeError` gives +// it, so it carries the reason Node's `fetch` puts on `cause`. A 401 from the +// server means it no longer accepts the saved session: that prints one line +// saying to log in again and exits 3, as a missing or corrupt session file +// does. export const run = async ( command: Promise, stdout: Writable, @@ -26,9 +29,7 @@ export const run = async ( ); code = 3; } else { - stderr.write( - `quak: ${err instanceof Error ? err.message : String(err)}\n`, - ); + stderr.write(`quak: ${describeError(err)}\n`); code = 1; } } diff --git a/src/errors.ts b/src/errors.ts index e001092..f6ac921 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,10 +1,13 @@ -// Error types that more than one layer of quak needs to recognise. +// Error types that more than one layer of quak needs to recognise, and how an +// error is described in what quak prints and in `failures.json`. // -// They live here rather than beside the code that throws them so that the +// The types live here rather than beside the code that throws them so that the // retry classifier can identify them without importing the HTTP client or the // download layer — both of which import the classifier. `ApiError` is // re-exported from `src/api/client.ts`, which is where callers have always -// imported it from and where it still belongs conceptually. +// imported it from and where it still belongs conceptually. The walk along an +// error's `cause` chain is here too, because the retry classifier reads the +// chain for errnos and `describeError` for messages. export class ApiError extends Error { readonly status: number; @@ -43,3 +46,45 @@ export class TruncatedStreamError extends Error { this.name = "TruncatedStreamError"; } } + +// `cause` is an arbitrary user-settable property and nothing prevents it from +// forming a cycle, so the walk is bounded. Hanging the process would be worse +// than misclassifying an error or describing only part of it. +const MAX_CAUSE_DEPTH = 8; + +// The error and each error on its `cause` chain, top first, at most +// `MAX_CAUSE_DEPTH` of them. `complete` is false when the walk stopped at the +// depth limit with more of the chain still below it. +export const causeChain = ( + err: unknown, +): { chain: object[]; complete: boolean } => { + const chain: object[] = []; + let current: unknown = err; + for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) { + if (current === null || typeof current !== "object") { + return { chain, complete: true }; + } + chain.push(current); + const { cause } = current as { cause?: unknown }; + if (cause === current) return { chain, complete: true }; + current = cause; + } + return { chain, complete: current === null || typeof current !== "object" }; +}; + +// An error as quak prints it and records it in `failures.json`: its message, +// then the message of each error on its `cause` chain, joined with ": ". Node's +// `fetch` rejects with just "fetch failed" and puts the reason on `cause`, so +// this gives "fetch failed: connect ECONNREFUSED 1.2.3.4:443". A cause whose +// message is already in the text before it is left out. +export const describeError = (err: unknown): string => { + if (!(err instanceof Error)) return String(err); + let text = err.message; + for (const cause of causeChain(err).chain.slice(1)) { + const { message } = cause as { message?: unknown }; + if (typeof message === "string" && !text.includes(message)) { + text += `: ${message}`; + } + } + return text; +}; diff --git a/src/retry.ts b/src/retry.ts index ffa68d9..09bb752 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -10,7 +10,7 @@ // declining to retry a transient one costs a single file that the next run // picks up anyway. When the evidence is ambiguous, fail fast. -import { ApiError, TruncatedStreamError } from "./errors.js"; +import { ApiError, causeChain, TruncatedStreamError } from "./errors.js"; export interface RetryOptions { // Total calls, not retries: `attempts: 1` disables retrying. @@ -89,11 +89,6 @@ const TRANSPORT_CODES = new Set([ // narrowed. See `isSafeToReplay`. const CONNECT_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "ECONNREFUSED"]); -// `cause` is an arbitrary user-settable property and nothing prevents it from -// forming a cycle, so the walk is bounded. Hanging the process would be a -// worse outcome than any misclassification. -const MAX_CAUSE_DEPTH = 8; - // Collect every `code` in an error's cause chain. undici does not put the // errno on the error it throws — it hangs the underlying socket error off // `cause`, sometimes more than one level down — so a classifier that only read @@ -101,20 +96,21 @@ const MAX_CAUSE_DEPTH = 8; // connection permanent. `complete` is false when the walk stopped at the // depth limit with more of the chain still below it. const causeCodes = (err: unknown): { codes: string[]; complete: boolean } => { + const { chain, complete } = causeChain(err); const codes: string[] = []; - let current: unknown = err; - for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) { - if (current === null || typeof current !== "object") { - return { codes, complete: true }; - } - const { code, cause } = current as { code?: unknown; cause?: unknown }; + for (const link of chain) { + const { code } = link as { code?: unknown }; if (typeof code === "string") codes.push(code); - if (cause === current) return { codes, complete: true }; - current = cause; } - return { codes, complete: current === null || typeof current !== "object" }; + return { codes, complete }; }; +// Does the error's cause chain hold the errno of a transport failure, one of +// `TRANSPORT_CODES`? `isRetryable` retries such an error, and `classify` in +// `src/backup.ts` calls it transient. +export const hasTransportCode = (err: unknown): boolean => + causeCodes(err).codes.some((code) => TRANSPORT_CODES.has(code)); + const isAbort = (err: unknown): boolean => { if (err === null || typeof err !== "object") return false; const { name } = err as { name?: unknown }; @@ -158,7 +154,7 @@ export const isRetryable = (err: unknown): boolean => { // have succeeded; the cost of the imprecision is bounded by the attempt // count. if (err instanceof TypeError) return true; - return causeCodes(err).codes.some((code) => TRANSPORT_CODES.has(code)); + return hasTransportCode(err); }; // Could the first attempt already have taken effect on the server? diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index 65bf5b6..8a14b66 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -329,6 +329,16 @@ const downloadOf = (input: RequestInfo | URL): number | undefined => { : undefined; }; +// What Node's `fetch` rejects with when the connection is refused, as for +// every failure below HTTP: a TypeError that says only "fetch failed", with +// the reason, and its errno, as its cause. +const refused = (): TypeError => + new TypeError("fetch failed", { + cause: Object.assign(new Error("connect ECONNREFUSED 1.2.3.4:443"), { + code: "ECONNREFUSED", + }), + }); + // The fake server, pushing onto `downloads` the ID of each file whose original // is downloaded. const recordingDownloads = @@ -619,6 +629,67 @@ describe("quak backup", () => { expect(readdirSync(dir)).toEqual([]); }); + // The server refuses every connection. quak tries each request again, + // here without waiting, and then the run stops at the refresh it starts + // with. The line it prints gives the reason Node's `fetch` put on the + // error's cause, so a cron log tells a refused connection from a name + // that did not resolve, and the exit code is 1. + it("prints why the refresh could not reach the server, and exits 1", async () => { + const dir = mkdtempSync(join(root, "backup-")); + const fetch: typeof globalThis.fetch = async () => { + throw refused(); + }; + + const { code, stderr } = await backup(dir, {}, fetch); + + expect(code).toBe(1); + expect(stderr).toBe( + "Starting backup...\n" + + "Refreshing library...\n" + + "quak: fetch failed: connect ECONNREFUSED 1.2.3.4:443\n", + ); + }); + + // The files CDN refuses every connection for toast.mp4. Once its retries + // are spent, the file goes into `failures.json` with the reason, as a + // transient failure, since a later run may well reach the CDN, and the + // lines the backup prints for it give the same reason. The rest of the + // account is backed up, and the exit code is 1. + it("records why a download could not reach the server in failures.json, as transient, and exits 1", async () => { + let attempts = 0; + const fetch: typeof globalThis.fetch = async (input, init) => { + if (downloadOf(input) === 200) { + attempts++; + throw refused(); + } + return server(input, init); + }; + const dir = mkdtempSync(join(root, "backup-")); + + const { code, stderr } = await backup(dir, {}, fetch); + + expect(code).toBe(1); + expect(attempts).toBe(UNATTENDED_RETRY_OPTIONS.attempts); + const error = "fetch failed: connect ECONNREFUSED 1.2.3.4:443"; + const failures = readJSON(join(dir, "failures.json")); + expect(Object.keys(failures.files)).toEqual(["200"]); + expect(failures.files["200"]).toMatchObject({ + fileID: 200, + title: "toast.mp4", + classification: "transient", + attempts: 1, + error, + }); + expect(stderr).toContain(`FAILED original toast.mp4: ${error}\n`); + expect(stderr).toContain( + `[Friend's Wedding] toast.mp4 (id 200): ${error}\n`, + ); + expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual( + Buffer.from(JPEG_WITH_EXIF), + ); + expect(existsSync(join(dir, DAY, "2024-03-01.200.mp4"))).toBe(false); + }); + // Something changed beach.jpg on disk after it was backed up. A run with // --verify hashes every stored original, finds that one no longer matches // the hash Ente recorded, says so, and downloads it again; it put back the diff --git a/test/cli/run.test.ts b/test/cli/run.test.ts index cfad348..fa1fc0e 100644 --- a/test/cli/run.test.ts +++ b/test/cli/run.test.ts @@ -48,6 +48,35 @@ describe("run", () => { ); }); + // Node's `fetch` rejects every failure below HTTP with a TypeError that + // says only "fetch failed", and hangs the reason, with its errno, off + // `cause`. The printed line carries the reason, so a cron log tells a + // refused connection from a name that did not resolve. + it("prints the reason a fetch failed, from the error's cause", async () => { + const refused = new TypeError("fetch failed", { + cause: Object.assign( + new Error("connect ECONNREFUSED 1.2.3.4:443"), + { code: "ECONNREFUSED" }, + ), + }); + const result = await runToExit(Promise.reject(refused)); + expect(result).toEqual({ + code: 1, + stdout: "", + stderr: "quak: fetch failed: connect ECONNREFUSED 1.2.3.4:443\n", + }); + }); + + // An error that already quotes its cause's message is printed as it is. + it("does not print a cause's message twice", async () => { + const reset = new Error("read ECONNRESET"); + const wrapped = new Error(`download: file 7: ${reset.message}`, { + cause: reset, + }); + const result = await runToExit(Promise.reject(wrapped)); + expect(result.stderr).toBe("quak: download: file 7: read ECONNRESET\n"); + }); + it("prints a thrown value that is not an Error", async () => { const result = await runToExit(Promise.reject("offline")); expect(result).toEqual({