Add failing tests for the download retry policy

Tests only; the retry module they import does not exist yet, so the
branch is red at this commit.

New test/retry/retry.test.ts documents the classifier and the backoff:
which errors are worth another attempt, which are not, and how the delay
before each retry is derived. It asserts on the arguments handed to an
injected sleep function rather than on elapsed time, so the suite never
waits and the numbers are exact.

test/api/client.test.ts gains the request-count contract for each of the
six call sites, the deadline behaviour, the ApiError typing that the
presigned PUT and the null-body paths need in order to be classified at
all, and the replay rule for the two non-idempotent methods.

test/download/download.test.ts gains the case that motivates the whole
design: a socket reset after the response headers arrived, which happens
below ApiClient and can only be caught by retrying the request, the
stream consumption and the decryption together. It also pins that a
retried download stages exactly one temp file, and that the two retry
layers do not compose into a multiplied request budget. The existing
truncation tests now assert on the error type rather than its wording,
since that type is what the classifier reads.

test/thumbnails/thumbnails.test.ts separates a genuine 404 from an
exhausted retry, so a failing server can no longer make
fix-missing-thumbnails re-upload thumbnails that already exist.
This commit is contained in:
2026-08-09 05:11:07 +00:00
parent 937bcb7aee
commit 0cbe338b58
5 changed files with 1535 additions and 16 deletions

View File

@@ -29,12 +29,23 @@
* return a `ReadableStream<Uint8Array>` from the appropriate CDN
* (or the self-hosted fallback path).
*
* - Retries and timeouts. Every request is issued under a deadline and,
* where it is safe to do so, retried with exponential backoff. The
* policy is `src/retry.ts`; what the last section of this file
* documents is which requests get it and which deliberately do not.
*
* All tests inject a fake `fetch` via the constructor so nothing touches
* the network. The fake records every call for assertion.
*/
import { describe, expect, it } from "vitest";
import { ApiClient, ApiError } from "../../src/api/client.js";
import {
ApiClient,
ApiError,
DEFAULT_DOWNLOAD_TIMEOUT_MS,
DEFAULT_REQUEST_TIMEOUT_MS,
} from "../../src/api/client.js";
import type { RetryOptions } from "../../src/retry.js";
// ---------------------------------------------------------------------------
// Test helpers
@@ -102,6 +113,92 @@ const recordingFetch = (
return { fetch: fake as typeof globalThis.fetch, calls };
};
/**
* One scripted outcome for a single `fetch` call:
*
* - a `Response`, returned as-is;
* - an `Error`, thrown — this is how `fetch` reports a network failure;
* - `HANG`, a request that never answers until its own deadline aborts it.
*
* `HANG` is what makes the timeout tests honest. A fake that resolved after a
* delay would be testing the clock; this one resolves *only* when the signal
* quak attached fires. If no signal is attached, or the signal is not wired to
* the body, the promise never settles and the test fails on its own timeout
* rather than passing by accident.
*/
const HANG = Symbol("hang until aborted");
type FetchStep = Response | Error | typeof HANG;
const scriptedFetch = (
...steps: FetchStep[]
): {
fetch: typeof globalThis.fetch;
calls: { url: string; init: RequestInit | undefined }[];
} => {
const calls: { url: string; init: RequestInit | undefined }[] = [];
let i = 0;
const fake = async (
input: RequestInfo | URL,
init?: RequestInit,
): Promise<Response> => {
const url =
typeof input === "string"
? input
: input instanceof URL
? input.href
: input.url;
calls.push({ url, init });
const step = steps[i++];
if (step === undefined) {
throw new Error(`scriptedFetch: no step for call #${i - 1}`);
}
if (step === HANG) {
return new Promise<Response>((_resolve, reject) => {
const signal = init?.signal;
if (!signal) return;
if (signal.aborted) {
reject(signal.reason as Error);
return;
}
signal.addEventListener(
"abort",
() => reject(signal.reason as Error),
{ once: true },
);
});
}
if (step instanceof Error) throw step;
return step;
};
return { fetch: fake as typeof globalThis.fetch, calls };
};
/** An error shaped like a Node transport failure: the errno is on `.code`. */
const errnoError = (code: string, message = code): Error =>
Object.assign(new Error(message), { code });
/**
* A retry policy with the waiting removed. Backoff arithmetic is covered in
* `test/retry/retry.test.ts`; what the tests below are about is *how many
* requests* each call site issues, so they inject a `sleep` that returns
* immediately. Nothing in this file waits.
*/
const noWait: RetryOptions = {
sleep: () => Promise.resolve(),
random: () => 0,
};
/** Drain a stream and return the bytes, so body-level failures surface. */
const readAll = async (stream: ReadableStream<Uint8Array>): Promise<number> => {
const reader = stream.getReader();
let total = 0;
for (;;) {
const { done, value } = await reader.read();
if (value) total += value.length;
if (done) return total;
}
};
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
@@ -383,3 +480,445 @@ describe("ApiClient.getFileStream / getThumbnailStream", () => {
expect(headers.get("X-Auth-Token")).toBe("tk");
});
});
// ---------------------------------------------------------------------------
// Retries
//
// The rule the whole section turns on: a request is repeated only when
// repeating it could produce a different answer, and only when repeating it
// cannot do harm. Those are two separate questions and the second one is why
// `postJSON` and `putJSON` behave differently from everything else here.
//
// Every assertion below counts requests. None of them measures how long
// anything took: the retry policy's `sleep` is injected and returns
// immediately, so a machine under load and an idle one produce identical
// results.
// ---------------------------------------------------------------------------
describe("ApiClient retries", () => {
it("issues exactly one request for a 404", async () => {
// A 404 is an answer, not a failure to get one. Repeating it wastes
// a round trip and — for `listMissingThumbnails`, which reads a 404
// as "this thumbnail really is missing" — delays a correct result.
// The script holds five responses; only the first may be consumed.
const { fetch, calls } = scriptedFetch(
textResponse("Not Found", 404),
textResponse("Not Found", 404),
textResponse("Not Found", 404),
textResponse("Not Found", 404),
textResponse("Not Found", 404),
);
const client = new ApiClient({ fetch, retry: noWait });
await expect(client.getJSON("/missing")).rejects.toBeInstanceOf(
ApiError,
);
expect(calls).toHaveLength(1);
});
it("retries a 500 up to the configured attempt count, then throws", async () => {
// `attempts` is a total, not a number of retries: three attempts mean
// three requests. The script offers five responses so that a client
// which ignored the limit would be visible as a count of 4 or 5
// rather than as a crash.
const { fetch, calls } = scriptedFetch(
textResponse("boom", 500),
textResponse("boom", 500),
textResponse("boom", 500),
textResponse("boom", 500),
textResponse("boom", 500),
);
const client = new ApiClient({
fetch,
retry: { ...noWait, attempts: 3 },
});
const err: unknown = await client
.getJSON("/flaky")
.catch((e: unknown) => e);
expect(err).toBeInstanceOf(ApiError);
expect((err as ApiError).status).toBe(500);
expect(calls).toHaveLength(3);
});
it("returns the first successful response after a 503", async () => {
const { fetch, calls } = scriptedFetch(
textResponse("unavailable", 503),
jsonResponse({ ok: true }),
);
const client = new ApiClient({ fetch, retry: noWait });
await expect(
client.getJSON<{ ok: boolean }>("/health"),
).resolves.toEqual({ ok: true });
expect(calls).toHaveLength(2);
});
it("retries 408 and 429", async () => {
// The two 4xx codes that are about timing rather than about the
// request. Backoff is precisely the right response to both.
for (const status of [408, 429]) {
const { fetch, calls } = scriptedFetch(
textResponse("wait", status),
jsonResponse({ ok: true }),
);
const client = new ApiClient({ fetch, retry: noWait });
await expect(
client.getJSON("/rate-limited"),
).resolves.toBeDefined();
expect(calls).toHaveLength(2);
}
});
it("retries a fetch rejection and succeeds on a later attempt", async () => {
// A dropped or refused connection is the failure this policy exists
// for: the request never got an answer, so asking again is free of
// consequence and likely to work.
const { fetch, calls } = scriptedFetch(
new TypeError("fetch failed"),
errnoError("ECONNRESET", "read ECONNRESET"),
jsonResponse({ collections: [] }),
);
const client = new ApiClient({ fetch, retry: noWait });
await expect(client.getJSON("/collections/v2")).resolves.toEqual({
collections: [],
});
expect(calls).toHaveLength(3);
});
it("applies the policy to file and thumbnail streams", async () => {
// Both CDN endpoints go through the same wrapper. A 503 from
// files.ente.io during a large backup is common enough that not
// retrying it would fail files for no reason.
for (const get of ["getFileStream", "getThumbnailStream"] as const) {
const { fetch, calls } = scriptedFetch(
textResponse("unavailable", 503),
streamResponse(new Uint8Array([1, 2, 3])),
);
const client = new ApiClient({ fetch, retry: noWait });
const stream = await client[get](7);
expect(await readAll(stream)).toBe(3);
expect(calls).toHaveLength(2);
}
});
it("does not retry when the caller opts out", async () => {
// `{ retry: false }` exists for one caller: the download layer, which
// wraps request *and* body consumption *and* decryption in a single
// retry of its own. Without the opt-out the two budgets would
// multiply — four attempts each becoming sixteen requests for one
// file.
const { fetch, calls } = scriptedFetch(
textResponse("unavailable", 503),
streamResponse(new Uint8Array([1, 2, 3])),
);
const client = new ApiClient({ fetch, retry: noWait });
await expect(
client.getFileStream(7, { retry: false }),
).rejects.toBeInstanceOf(ApiError);
expect(calls).toHaveLength(1);
});
it("exposes its resolved retry policy to the download layer", async () => {
// The download layer runs its own `withRetry` and must run it under
// the same policy the client was configured with, not under the
// library defaults.
const { fetch } = scriptedFetch(jsonResponse({}));
const client = new ApiClient({
fetch,
retry: { ...noWait, attempts: 9, baseDelayMs: 7, maxDelayMs: 11 },
});
const policy = client.getRetryOptions();
expect(policy.attempts).toBe(9);
expect(policy.baseDelayMs).toBe(7);
expect(policy.maxDelayMs).toBe(11);
});
});
describe("ApiClient timeouts", () => {
it("ships bounded default deadlines", () => {
// Asserted here so the README and the code cannot drift. Two numbers
// rather than one, because a deadline that is sane for a JSON call is
// nowhere near enough for a multi-gigabyte body, and a deadline long
// enough for that body would let a hung API call stall a backup for
// ten minutes.
expect(DEFAULT_REQUEST_TIMEOUT_MS).toBe(30_000);
expect(DEFAULT_DOWNLOAD_TIMEOUT_MS).toBe(600_000);
});
it("attaches an abort signal to every request", async () => {
const { fetch, calls } = recordingFetch(
jsonResponse({}),
jsonResponse({}),
new Response(null, { status: 200 }),
jsonResponse({}),
);
const client = new ApiClient({ fetch, retry: noWait });
await client.getJSON("/a");
await client.postJSON("/b", {});
await client.putFile("https://s3.example/x", new Uint8Array([1]));
await client.putJSON("/c", {});
for (const call of calls) {
expect(call.init?.signal).toBeInstanceOf(AbortSignal);
}
});
it("gives up on a request that never answers, and retries it", async () => {
// Before this policy existed there was no timeout anywhere in quak: a
// CDN connection that accepted the request and then went quiet would
// hang `quak backup` forever. `HANG` reproduces exactly that — the
// fake never answers, so the only thing that can end the call is the
// deadline quak attached.
const { fetch, calls } = scriptedFetch(HANG, HANG, HANG);
const client = new ApiClient({
fetch,
requestTimeoutMs: 20,
retry: { ...noWait, attempts: 3 },
});
await expect(client.getJSON("/black-hole")).rejects.toThrow();
// A timeout is retryable, so all three attempts were spent...
expect(calls).toHaveLength(3);
// ...each under its own fresh deadline, not one shared one that
// expired during the first attempt.
const signals = calls.map((c) => c.init?.signal);
expect(new Set(signals).size).toBe(3);
}, 5000);
it("recovers when a later attempt answers in time", async () => {
const { fetch, calls } = scriptedFetch(HANG, jsonResponse({ ok: 1 }));
const client = new ApiClient({
fetch,
requestTimeoutMs: 20,
retry: noWait,
});
await expect(client.getJSON("/slow-then-fast")).resolves.toEqual({
ok: 1,
});
expect(calls).toHaveLength(2);
}, 5000);
it("aborts a body that stalls after the headers arrived", async () => {
// The failure mode that a naive timeout misses. `getFileStream`
// returns as soon as headers arrive; the bytes are pulled later, in
// the download layer. A deadline that only guarded the initial fetch
// would leave the identical hang one layer down — which is where
// multi-megabyte photo downloads actually stall.
//
// This fake resolves its headers immediately and then serves a body
// that never produces a chunk and never observes the signal, so the
// only thing that can unblock the read is quak's own enforcement of
// the deadline over the stream it hands out.
const stalling = new Response(
new ReadableStream<Uint8Array>({
pull: () => new Promise<void>(() => {}),
}),
{ status: 200 },
);
const { fetch } = scriptedFetch(stalling);
const client = new ApiClient({
fetch,
downloadTimeoutMs: 20,
retry: { ...noWait, attempts: 1 },
});
const stream = await client.getFileStream(42);
const err: unknown = await readAll(stream).catch((e: unknown) => e);
expect(err).toBeInstanceOf(Error);
expect((err as Error).name).toBe("TimeoutError");
}, 5000);
it("lets a body that arrives in time through untouched", async () => {
// The counterpart to the previous test: enforcing the deadline over
// the stream must not corrupt or truncate a body that is simply being
// read normally.
const payload = new Uint8Array([9, 8, 7, 6, 5]);
const { fetch } = scriptedFetch(streamResponse(payload));
const client = new ApiClient({ fetch, retry: noWait });
const stream = await client.getFileStream(42);
const reader = stream.getReader();
const chunks: Uint8Array[] = [];
for (;;) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
}
const joined = new Uint8Array(chunks.reduce((n, c) => n + c.length, 0));
let offset = 0;
for (const c of chunks) {
joined.set(c, offset);
offset += c.length;
}
expect(joined).toEqual(payload);
});
});
describe("ApiClient error typing", () => {
it("throws ApiError with the status when a presigned PUT fails", async () => {
// `putFile` used to throw a bare Error with the status baked into a
// string. Nothing downstream could classify it, so a 500 from S3 was
// indistinguishable from a bug and could never be retried.
const { fetch, calls } = scriptedFetch(
new Response("Forbidden", { status: 403 }),
);
const client = new ApiClient({ fetch, retry: noWait });
const err: unknown = await client
.putFile("https://s3.example/obj", new Uint8Array([1, 2]))
.catch((e: unknown) => e);
expect(err).toBeInstanceOf(ApiError);
expect((err as ApiError).status).toBe(403);
expect(calls).toHaveLength(1);
});
it("retries a presigned PUT on a 5xx", async () => {
// A presigned PUT writes the whole object at one key in one request,
// so repeating it either overwrites the same bytes or lands them for
// the first time. There is no partial state to protect.
const { fetch, calls } = scriptedFetch(
new Response("slow down", { status: 503 }),
new Response(null, { status: 200 }),
);
const client = new ApiClient({ fetch, retry: noWait });
await client.putFile("https://s3.example/obj", new Uint8Array([1, 2]));
expect(calls).toHaveLength(2);
});
it("throws ApiError when a download response has no body", async () => {
// Also previously a bare Error. It carries the response status so a
// caller can see what arrived — and it is *not* retried: a 200 with
// no body is a malformed response, and asking again produces the same
// malformed response.
for (const get of ["getFileStream", "getThumbnailStream"] as const) {
const { fetch, calls } = scriptedFetch(
new Response(null, { status: 200 }),
new Response(null, { status: 200 }),
);
const client = new ApiClient({ fetch, retry: noWait });
const err: unknown = await client[get](5).catch((e: unknown) => e);
expect(err).toBeInstanceOf(ApiError);
expect((err as ApiError).status).toBe(200);
expect(calls).toHaveLength(1);
}
});
});
describe("ApiClient non-idempotent requests", () => {
/**
* `postJSON` and `putJSON` carry quak's only requests that change server
* state: `/users/srp/create-session`, `/users/two-factor/verify` — which
* consumes one of a small number of 2FA attempts — and `/files/thumbnail`.
*
* They are retried only when the failure proves the request never reached
* the server, which in practice means the connection was never
* established. Everything else is ambiguous: a 5xx proves the server did
* process the request, and a reset or a timeout can arrive after it did.
* Replaying under that ambiguity can burn a 2FA attempt or register a
* thumbnail twice, and neither is worth the round trip it saves.
*/
it("does not replay a POST after a 5xx", async () => {
const { fetch, calls } = scriptedFetch(
textResponse("boom", 500),
jsonResponse({ sessionID: "second" }),
);
const client = new ApiClient({ fetch, retry: noWait });
await expect(
client.postJSON("/users/two-factor/verify", { code: "123456" }),
).rejects.toBeInstanceOf(ApiError);
expect(calls).toHaveLength(1);
});
it("does not replay a POST after a mid-flight connection reset", async () => {
// A reset can happen after the request was fully sent and acted on.
// It is retryable in general — `getJSON` retries it — but it is not
// replay-safe.
const { fetch, calls } = scriptedFetch(
errnoError("ECONNRESET", "socket hang up"),
jsonResponse({ sessionID: "second" }),
);
const client = new ApiClient({ fetch, retry: noWait });
await expect(
client.postJSON("/users/srp/create-session", {}),
).rejects.toThrow(/ECONNRESET|socket hang up/);
expect(calls).toHaveLength(1);
});
it("does not replay a POST after a timeout", async () => {
// A deadline says nothing about whether the server acted.
const { fetch, calls } = scriptedFetch(HANG, jsonResponse({}));
const client = new ApiClient({
fetch,
requestTimeoutMs: 20,
retry: noWait,
});
await expect(client.postJSON("/users/ott", {})).rejects.toThrow();
expect(calls).toHaveLength(1);
}, 5000);
it("replays a POST when the connection was never established", async () => {
// A refused connection or a DNS failure happens before any request
// byte is written, so the server cannot have seen it. This is the one
// case where replaying is provably harmless.
const { fetch, calls } = scriptedFetch(
new TypeError("fetch failed", {
cause: errnoError("ECONNREFUSED", "connect ECONNREFUSED"),
}),
errnoError("EAI_AGAIN", "getaddrinfo EAI_AGAIN api.ente.io"),
jsonResponse({ sessionID: "third" }),
);
const client = new ApiClient({ fetch, retry: noWait });
await expect(
client.postJSON<{ sessionID: string }>(
"/users/srp/create-session",
{},
),
).resolves.toEqual({ sessionID: "third" });
expect(calls).toHaveLength(3);
});
it("applies the same rule to PUT", async () => {
// `/files/thumbnail` is reached through `putJSON`.
const failing = scriptedFetch(
textResponse("boom", 503),
jsonResponse({}),
);
const failingClient = new ApiClient({
fetch: failing.fetch,
retry: noWait,
});
await expect(
failingClient.updateThumbnail(1, "key", "header"),
).rejects.toBeInstanceOf(ApiError);
expect(failing.calls).toHaveLength(1);
const refused = scriptedFetch(
errnoError("ECONNREFUSED", "connect ECONNREFUSED"),
jsonResponse({}),
);
const refusedClient = new ApiClient({
fetch: refused.fetch,
retry: noWait,
});
await refusedClient.updateThumbnail(1, "key", "header");
expect(refused.calls).toHaveLength(2);
});
});