/** * Tests for `src/retry.ts` — the retry policy shared by every network * operation in quak. * * Two things live in that module and they are deliberately separate: * * - **`isRetryable(err)`**, a pure classifier. Given an error, is trying * again capable of producing a different answer? Nothing else about the * error matters: not how it was logged, not where it came from. * * - **`withRetry(fn, opts)`**, the loop. It calls `fn`, and while the error * is classified retryable and attempts remain, it sleeps and calls `fn` * again. It never inspects errors itself. * * The classifier's default answer is *no*. quak is a backup tool: a wrongly * retried permanent failure costs a user round trips and delays the rest of * the run, while a wrongly rejected transient failure costs one file that the * next run picks up. When in doubt, fail fast. * * ## Reading the backoff assertions * * `withRetry` takes its `sleep` and its `random` as injected functions. Every * test here passes a `sleep` that records the delay it was asked for and * returns immediately, so the suite never waits, and a `random` that returns a * fixed number, so jitter is exact rather than approximate. **No assertion in * this file (or anywhere else in the suite) is about elapsed wall-clock time.** * They are about what `withRetry` asked for, and how many times `fn` ran. * * The delay before retry number *n* (1-based) is: * * random() * min(maxDelayMs, baseDelayMs * 2 ** (n - 1)) * * That is exponential backoff with full jitter: the exponential term is the * *ceiling*, and the actual wait is drawn uniformly below it. Full jitter, * rather than a fixed delay plus noise, is what stops a client that lost a * hundred parallel downloads to one CDN blip from re-sending all hundred at * the same instant. */ import { describe, expect, it } from "vitest"; import { DEFAULT_RETRY_OPTIONS, isRetryable, isSafeToReplay, resolveRetryOptions, withRetry, } from "../../src/retry.js"; import { ApiError, TruncatedStreamError } from "../../src/errors.js"; import { ApiError as ApiErrorFromClient } from "../../src/api/client.js"; // --------------------------------------------------------------------------- // Test helpers // --------------------------------------------------------------------------- /** * A `sleep` that records what it was asked to wait for and returns * immediately. This is the whole reason `withRetry` takes an injected sleep: * the retry policy is exercised in full — every branch, every delay — without * the suite spending a single millisecond waiting. */ const recordingSleep = (): { sleep: (ms: number) => Promise; delays: number[]; } => { const delays: number[] = []; return { sleep: (ms: number): Promise => { delays.push(ms); return Promise.resolve(); }, delays, }; }; /** An error shaped like a Node transport failure: the errno is on `.code`. */ const errnoError = (code: string, message = code): Error => Object.assign(new Error(message), { code }); // --------------------------------------------------------------------------- // Classification // --------------------------------------------------------------------------- describe("isRetryable: HTTP status codes", () => { it("does not retry ordinary 4xx responses", () => { // A 4xx is the server saying the request itself is wrong. Repeating // it verbatim produces the same answer, so retrying only delays the // failure the caller has to handle. 404 is the load-bearing case: // `listMissingThumbnails` depends on a 404 arriving promptly and // exactly once. for (const status of [400, 401, 403, 404, 409, 410, 422]) { expect(isRetryable(new ApiError(`HTTP ${status}`, status))).toBe( false, ); } }); it("retries 408 and 429", () => { // The two 4xx codes that are statements about timing rather than // about the request. 408 is the server admitting it gave up waiting; // 429 is it asking for less traffic — which backoff supplies. expect(isRetryable(new ApiError("timeout", 408))).toBe(true); expect(isRetryable(new ApiError("slow down", 429))).toBe(true); }); it("retries every 5xx response", () => { // A 5xx is the server failing, not the request being wrong. Ente's // CDN in particular returns 500 and 503 under load. for (const status of [500, 502, 503, 504, 599]) { expect(isRetryable(new ApiError(`HTTP ${status}`, status))).toBe( true, ); } }); it("treats a 2xx or 3xx ApiError as not retryable", () => { // These exist: `getFileStream` raises an ApiError carrying the // response status when a 200 arrives with a null body. That is a // malformed response, not a transport failure, and repeating the // request will produce the same malformed response. expect(isRetryable(new ApiError("response body is null", 200))).toBe( false, ); expect(isRetryable(new ApiError("redirect", 304))).toBe(false); }); it("uses the same ApiError class that ApiClient exports", () => { // `ApiError` lives in `src/errors.ts` and is re-exported from // `src/api/client.ts`, which is where every existing caller and test // imports it from. If those ever became two separate classes the // classifier would silently stop recognising errors raised by the // client, and every 5xx in the wild would be treated as permanent. expect(ApiErrorFromClient).toBe(ApiError); expect(new ApiErrorFromClient("boom", 503)).toBeInstanceOf(ApiError); expect(isRetryable(new ApiErrorFromClient("boom", 503))).toBe(true); }); }); describe("isRetryable: transport failures", () => { it("retries a TypeError, which is how fetch reports a failed request", () => { // Node's fetch rejects with `TypeError: fetch failed` for everything // below HTTP: DNS failure, refused connection, TLS error, reset // socket. The real diagnosis is on `cause`, but there is nothing on // the object that distinguishes it from a TypeError thrown by a bug, // so this rule is deliberately literal. The cost of the imprecision // is bounded by the attempt count; the alternative — demanding a // recognised `cause` — would classify real network failures as // permanent and fail backups that should have succeeded. expect(isRetryable(new TypeError("fetch failed"))).toBe(true); }); it("retries an errno carried on the error itself", () => { for (const code of [ "ECONNRESET", "ETIMEDOUT", "EPIPE", "ENOTFOUND", "EAI_AGAIN", "ECONNREFUSED", "EHOSTUNREACH", "ENETUNREACH", ]) { expect(isRetryable(errnoError(code))).toBe(true); } }); it("retries an errno buried in the cause chain", () => { // undici does not put the errno on the error it throws; it hangs the // underlying socket error off `cause`, sometimes more than one level // down. A classifier that only looked at the top-level error would // see a bare `Error` and call every dropped connection permanent. const nested = new Error("request to files.ente.io failed", { cause: new Error("socket hang up", { cause: errnoError("ECONNRESET", "read ECONNRESET"), }), }); expect(isRetryable(nested)).toBe(true); }); it("accepts a plain object as a cause", () => { // Not everything in a cause chain is an Error instance. expect( isRetryable(new Error("failed", { cause: { code: "ETIMEDOUT" } })), ).toBe(true); }); it("retries an aborted request", () => { // `AbortSignal.timeout()` aborts with a `TimeoutError`; an explicit // `abort()` produces an `AbortError`. quak only ever aborts a request // on its own deadline, so both mean "this attempt ran out of time", // which is exactly the condition a later attempt might not hit. expect(isRetryable(new DOMException("timed out", "TimeoutError"))).toBe( true, ); expect(isRetryable(new DOMException("aborted", "AbortError"))).toBe( true, ); }); it("does not confuse an unrelated errno with a transport failure", () => { // A filesystem error surfaces the same way an errno network error // does. Retrying a full disk or a missing directory is pointless. expect(isRetryable(errnoError("ENOSPC", "no space left"))).toBe(false); expect(isRetryable(errnoError("ENOENT", "no such file"))).toBe(false); expect(isRetryable(errnoError("EACCES", "permission denied"))).toBe( false, ); }); it("terminates on a cause chain that points at itself", () => { // Defensive: `cause` is an arbitrary user-settable property and // nothing stops it forming a cycle. Without a bound on the walk this // classifier would hang the process, which is a worse failure than // any misclassification. const looped: Error & { cause?: unknown } = new Error("loop"); looped.cause = looped; expect(isRetryable(looped)).toBe(false); }); }); describe("isRetryable: stream truncation versus corruption", () => { it("retries a truncated stream", () => { // Truncation is a transfer that stopped early. The bytes that did // arrive are useless, but the file on the server is fine, so asking // again is exactly right. expect(isRetryable(new TruncatedStreamError("stream truncated"))).toBe( true, ); }); it("retries a truncated stream whose cause is an authentication failure", () => { // The single-chunk ambiguity, recorded on issue #2 and inherited from // the truncation work: when a body ends part-way through a chunk, // Poly1305 fails and carries no framing signal, so a cut connection // and genuinely corrupt bytes are indistinguishable. That case is // reported as truncation with the authentication failure preserved as // `cause`, and it is therefore retried. // // Retrying is the deliberate choice. For a multi-chunk body the split // is real — a corrupt chunk mid-stream stays an authentication // failure, see the next test — but for a single-chunk body (most // thumbnails, every small file) a wrong key and a cut connection look // identical. The cost of guessing wrong is bounded: a few extra round // trips before the same failure. The cost of guessing the other way // is a silently truncated file kept forever. const err = new TruncatedStreamError("stream truncated", { cause: new Error("secretstream chunk authentication failed"), }); expect(isRetryable(err)).toBe(true); }); it("does not retry an authentication failure that is not truncation", () => { // A whole chunk that failed to authenticate while the stream carried // on past it cannot be a short transfer. It is corruption or a wrong // key, and no number of retries fixes either. expect( isRetryable(new Error("secretstream chunk authentication failed")), ).toBe(false); }); }); describe("isRetryable: everything else", () => { it("does not retry programming errors or unknown values", () => { expect(isRetryable(new Error("boom"))).toBe(false); expect(isRetryable(new RangeError("out of range"))).toBe(false); expect(isRetryable(new SyntaxError("bad JSON"))).toBe(false); expect(isRetryable("a string")).toBe(false); expect(isRetryable(undefined)).toBe(false); expect(isRetryable(null)).toBe(false); }); }); // --------------------------------------------------------------------------- // The replay-safety classifier for non-idempotent requests // --------------------------------------------------------------------------- describe("isSafeToReplay", () => { /** * `isRetryable` answers "could a retry succeed?". For a POST or a PUT * that is not the whole question: the other half is "could the first * attempt already have taken effect on the server?". * * quak's non-idempotent calls are `/users/srp/create-session`, * `/users/two-factor/verify` (which consumes one of a limited number of * 2FA attempts) and `/files/thumbnail`. A blind replay of any of them can * do real damage, so they retry only on failures that prove no request * byte ever reached the server — which means the connection was never * established. */ it("replays only failures where the connection was never established", () => { for (const code of [ "ENOTFOUND", "EAI_AGAIN", "ECONNREFUSED", "EHOSTUNREACH", "ENETUNREACH", ]) { expect(isSafeToReplay(errnoError(code))).toBe(true); } // Also when undici has buried it, which is how it actually arrives. expect( isSafeToReplay( new TypeError("fetch failed", { cause: errnoError("ECONNREFUSED"), }), ), ).toBe(true); }); it("does not replay a failure that could have happened after the server acted", () => { // Every one of these is ambiguous about whether the server processed // the request. A 5xx proves it did. A reset or a broken pipe can // arrive after the request was fully sent and handled. A timeout says // nothing at all about the server's state. A bare `fetch failed` with // no recognisable cause could be any of them. expect(isSafeToReplay(new ApiError("HTTP 500", 500))).toBe(false); expect(isSafeToReplay(new ApiError("HTTP 429", 429))).toBe(false); expect(isSafeToReplay(errnoError("ECONNRESET"))).toBe(false); expect(isSafeToReplay(errnoError("EPIPE"))).toBe(false); expect(isSafeToReplay(errnoError("ETIMEDOUT"))).toBe(false); expect( isSafeToReplay(new DOMException("timed out", "TimeoutError")), ).toBe(false); expect(isSafeToReplay(new TypeError("fetch failed"))).toBe(false); }); }); // --------------------------------------------------------------------------- // The retry loop // --------------------------------------------------------------------------- describe("withRetry", () => { it("calls the function once and does not sleep when it succeeds", async () => { const { sleep, delays } = recordingSleep(); let calls = 0; const result = await withRetry( () => { calls++; return Promise.resolve("ok"); }, { sleep }, ); expect(result).toBe("ok"); expect(calls).toBe(1); expect(delays).toEqual([]); }); it("stops at the first success and returns its value", async () => { const { sleep, delays } = recordingSleep(); let calls = 0; const result = await withRetry( () => { calls++; if (calls < 3) { return Promise.reject(new ApiError("HTTP 503", 503)); } return Promise.resolve(calls); }, { attempts: 5, sleep }, ); expect(result).toBe(3); expect(calls).toBe(3); // Two failures, so two waits — and none after the attempt that // succeeded. expect(delays).toHaveLength(2); }); it("gives up after `attempts` calls and throws the last error", async () => { // `attempts` counts calls, not retries: `attempts: 3` means the // function runs three times in total. The error that escapes is the // one from the final attempt, because that is the current state of // the world; a caller logging it is logging what is true now. const { sleep, delays } = recordingSleep(); let calls = 0; const failure = withRetry( () => { calls++; return Promise.reject(new ApiError(`attempt ${calls}`, 503)); }, { attempts: 3, sleep }, ); await expect(failure).rejects.toThrow("attempt 3"); expect(calls).toBe(3); // Three attempts, two gaps between them. Sleeping after the last // attempt would delay the caller's failure for nothing. expect(delays).toHaveLength(2); }); it("does not retry at all when attempts is 1", async () => { const { sleep, delays } = recordingSleep(); let calls = 0; await expect( withRetry( () => { calls++; return Promise.reject(new ApiError("HTTP 500", 500)); }, { attempts: 1, sleep }, ), ).rejects.toThrow("HTTP 500"); expect(calls).toBe(1); expect(delays).toEqual([]); }); it("rethrows a non-retryable error immediately", async () => { const { sleep, delays } = recordingSleep(); let calls = 0; await expect( withRetry( () => { calls++; return Promise.reject(new ApiError("HTTP 404", 404)); }, { attempts: 5, sleep }, ), ).rejects.toBeInstanceOf(ApiError); expect(calls).toBe(1); expect(delays).toEqual([]); }); it("preserves the error object, not just its message", async () => { // Callers classify what escapes: `listMissingThumbnails` needs the // `ApiError` and its status to tell a genuine 404 from a transient // failure. Wrapping the error in a "retries exhausted" error would // break that. const original = new ApiError("gone", 410, { code: "GONE" }); const err: unknown = await withRetry(() => Promise.reject(original), { sleep: () => Promise.resolve(), }).catch((e: unknown) => e); expect(err).toBe(original); }); it("honours a caller-supplied classifier", async () => { // This is how the non-idempotent call sites narrow the policy: same // loop, same backoff, stricter question. const { sleep } = recordingSleep(); let calls = 0; await expect( withRetry( () => { calls++; // Retryable under the default policy... return Promise.reject(new ApiError("HTTP 503", 503)); }, { attempts: 4, sleep, isRetryable: isSafeToReplay }, ), ).rejects.toThrow("HTTP 503"); // ...but not under `isSafeToReplay`, so it ran exactly once. expect(calls).toBe(1); }); }); describe("withRetry backoff", () => { it("doubles the ceiling on each retry and caps it", async () => { // `random: () => 1` pins the jitter to the top of its range, which // makes the ceiling itself observable. The sequence is // base, base*2, base*4, ... clamped at maxDelayMs — so a long outage // settles into a steady poll instead of growing to hours. const { sleep, delays } = recordingSleep(); await expect( withRetry(() => Promise.reject(new ApiError("HTTP 500", 500)), { attempts: 6, baseDelayMs: 100, maxDelayMs: 250, sleep, random: () => 1, }), ).rejects.toThrow(); expect(delays).toEqual([100, 200, 250, 250, 250]); }); it("draws each delay uniformly below its ceiling", async () => { // Full jitter. The exponential value is the maximum wait, not the // wait itself, so a fleet of clients that failed together does not // come back in lockstep. const { sleep, delays } = recordingSleep(); await expect( withRetry(() => Promise.reject(new ApiError("HTTP 500", 500)), { attempts: 4, baseDelayMs: 100, maxDelayMs: 10_000, sleep, random: () => 0.25, }), ).rejects.toThrow(); expect(delays).toEqual([25, 50, 100]); }); it("never asks to sleep longer than the cap or less than zero", async () => { // Whatever `random` returns from its [0, 1) contract, the delay stays // inside the configured envelope. const draws = [0, 0.999_999, 0.5, 0.1, 0.9]; let i = 0; const { sleep, delays } = recordingSleep(); await expect( withRetry(() => Promise.reject(new ApiError("HTTP 500", 500)), { attempts: 6, baseDelayMs: 1000, maxDelayMs: 2000, sleep, random: () => draws[i++]!, }), ).rejects.toThrow(); expect(delays).toHaveLength(5); for (const d of delays) { expect(d).toBeGreaterThanOrEqual(0); expect(d).toBeLessThanOrEqual(2000); } expect(delays[0]).toBe(0); }); }); describe("retry defaults", () => { it("ships a bounded, documented default policy", () => { // These are the numbers the README documents. They are asserted here // so the README and the code cannot drift apart silently: four // attempts, half a second of base delay, ten seconds of ceiling — // under 15 seconds of waiting in the worst case, which keeps a // several-thousand-file backup moving past a bad file rather than // stalling on it. expect(DEFAULT_RETRY_OPTIONS.attempts).toBe(4); expect(DEFAULT_RETRY_OPTIONS.baseDelayMs).toBe(500); expect(DEFAULT_RETRY_OPTIONS.maxDelayMs).toBe(10_000); }); it("fills in only the fields the caller left out", () => { const resolved = resolveRetryOptions({ attempts: 2 }); expect(resolved.attempts).toBe(2); expect(resolved.baseDelayMs).toBe(DEFAULT_RETRY_OPTIONS.baseDelayMs); expect(resolved.maxDelayMs).toBe(DEFAULT_RETRY_OPTIONS.maxDelayMs); expect(typeof resolved.sleep).toBe("function"); expect(typeof resolved.random).toBe("function"); }); it("resolves to the defaults when given nothing", () => { expect(resolveRetryOptions()).toEqual(DEFAULT_RETRY_OPTIONS); expect(resolveRetryOptions({})).toEqual(DEFAULT_RETRY_OPTIONS); }); it("defaults random to a real generator in [0, 1)", () => { const { random } = resolveRetryOptions(); for (let i = 0; i < 100; i++) { const r = random(); expect(r).toBeGreaterThanOrEqual(0); expect(r).toBeLessThan(1); } }); });