quak/src/retry.ts

// The retry policy shared by every network operation in quak.
//
// Two independent pieces: a classifier that decides whether an error is worth
// another attempt, and a loop that acts on that decision with exponential
// backoff. Keeping them apart is what lets the non-idempotent call sites reuse
// the loop under a stricter question (see `isSafeToReplay`).
//
// The classifier's default answer is no. For a backup tool, retrying a
// permanent failure spends round trips and delays every remaining file, while
// declining to retry a transient one costs a single file that the next run
// picks up anyway. When the evidence is ambiguous, fail fast.

import { ApiError, causeChain, TruncatedStreamError } from "./errors.js";

export interface RetryOptions {
    // Total calls, not retries: `attempts: 1` disables retrying.
    attempts?: number;
    // Ceiling for the first retry's delay; doubles with each retry.
    baseDelayMs?: number;
    // Upper bound on that ceiling, so a long outage settles into a steady
    // poll instead of growing without limit.
    maxDelayMs?: number;
    // Injected so tests exercise the whole policy without waiting.
    sleep?: (ms: number) => Promise<void>;
    // Injected so the jitter is reproducible under test.
    random?: () => number;
}

export type ResolvedRetryOptions = Required<RetryOptions>;

export const DEFAULT_RETRY_OPTIONS: ResolvedRetryOptions = {
    attempts: 4,
    baseDelayMs: 500,
    maxDelayMs: 10_000,
    sleep: (ms: number): Promise<void> =>
        new Promise((resolve) => setTimeout(resolve, ms)),
    random: Math.random,
};

// For a run nobody is watching, such as `quak backup` from cron. A request that
// keeps failing waits at most 243 s in all before it gives up, usually about
// half that, since each wait is drawn at random below its ceiling.
export const UNATTENDED_RETRY_OPTIONS: ResolvedRetryOptions = {
    ...DEFAULT_RETRY_OPTIONS,
    attempts: 10,
    baseDelayMs: 1_000,
    maxDelayMs: 60_000,
};

export const resolveRetryOptions = (
    opts?: RetryOptions,
): ResolvedRetryOptions => ({
    attempts: opts?.attempts ?? DEFAULT_RETRY_OPTIONS.attempts,
    baseDelayMs: opts?.baseDelayMs ?? DEFAULT_RETRY_OPTIONS.baseDelayMs,
    maxDelayMs: opts?.maxDelayMs ?? DEFAULT_RETRY_OPTIONS.maxDelayMs,
    sleep: opts?.sleep ?? DEFAULT_RETRY_OPTIONS.sleep,
    random: opts?.random ?? DEFAULT_RETRY_OPTIONS.random,
});

// Transport failures: the request did not complete, for reasons below HTTP.
//
// The errnos are Node's. The binary `make install` builds runs under bun,
// whose `fetch` rejects with a plain `Error` that has no `cause` and a `code`
// of bun's own. Bun gives `ConnectionRefused` for a refused connection and for
// a TLS handshake that failed, and `ConnectionRefused` or `FailedToOpenSocket`
// for a name that does not resolve. For a connection reset or closed before
// the response ended, it gives Node's `ECONNRESET`. These were found by
// running that binary, built with bun 1.3.13, against listeners on 127.0.0.1.
const TRANSPORT_CODES = new Set([
    "ECONNRESET",
    "ECONNABORTED",
    "ETIMEDOUT",
    "EPIPE",
    "ENOTFOUND",
    "EAI_AGAIN",
    "ECONNREFUSED",
    "EHOSTUNREACH",
    "ENETUNREACH",
    "ENETRESET",
    "ENETDOWN",
    // Bun's.
    "ConnectionRefused",
    "FailedToOpenSocket",
]);

// The subset of the above that can only be reported before a TCP connection
// exists, and therefore before any request byte could have been written: name
// resolution produced no address (`ENOTFOUND`, `EAI_AGAIN`) or the peer
// refused the connection with an RST to the SYN (`ECONNREFUSED`).
//
// The routing errnos — `EHOSTUNREACH`, `ENETUNREACH`, `ENETDOWN` — are
// deliberately absent even though they look like connect-time failures. On
// Linux they are also delivered on an already-established socket: an ICMP
// destination-unreachable arriving mid-flight sets the socket error and the
// next read or write returns it, and a local interface going down after the
// request was fully written surfaces the same way. In those cases the server
// may already have received and acted on the request, which is exactly the
// ambiguity this set exists to exclude. They stay in `TRANSPORT_CODES`, so
// they remain retryable for idempotent calls; only replay eligibility is
// narrowed. See `isSafeToReplay`.
//
// Bun's `ConnectionRefused` and `FailedToOpenSocket` are absent too, so a
// `POST` or `PUT` is not replayed on them. When a connection bun kept open from
// an earlier request closes before any answer, bun sends the request again on
// a new connection, and if that one fails it gives one of these codes although
// the server may already have the request.
const CONNECT_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "ECONNREFUSED"]);

// Collect every `code` in an error's cause chain. undici does not put the
// errno on the error it throws — it hangs the underlying socket error off
// `cause`, sometimes more than one level down — so a classifier that only read
// the top-level error would see a bare `Error` and call every dropped
// connection permanent. `complete` is false when the walk stopped at the
// depth limit with more of the chain still below it.
const causeCodes = (err: unknown): { codes: string[]; complete: boolean } => {
    const { chain, complete } = causeChain(err);
    const codes: string[] = [];
    for (const link of chain) {
        const { code } = link as { code?: unknown };
        if (typeof code === "string") codes.push(code);
    }
    return { codes, complete };
};

// Does the error's cause chain hold the code of a transport failure, one of
// `TRANSPORT_CODES`? `isRetryable` retries such an error, and `classify` in
// `src/backup.ts` calls it transient.
export const hasTransportCode = (err: unknown): boolean =>
    causeCodes(err).codes.some((code) => TRANSPORT_CODES.has(code));

const isAbort = (err: unknown): boolean => {
    if (err === null || typeof err !== "object") return false;
    const { name } = err as { name?: unknown };
    return name === "AbortError" || name === "TimeoutError";
};

// Is another attempt capable of producing a different answer?
//
// A note on the truncation case, recorded on issue #2. `TruncatedStreamError`
// is retried, and for a body of more than one chunk that is exactly right: a
// chunk that failed to authenticate while the stream carried on past it is
// corruption, stays an ordinary authentication failure, and is not retried.
//
// For a single-chunk body — most thumbnails, every small file — the split is
// not achievable. A wrong file key, server-side corruption and a connection
// cut mid-chunk are cryptographically identical: Poly1305 fails and carries no
// framing signal. All three are reported as truncation and therefore retried.
// That imprecision is deliberate and bounded by the attempt count: one wasted
// round trip on a genuinely corrupt file is a fair price for never silently
// keeping a truncated one, and the alternative — treating a single-chunk
// authentication failure as permanent — would reintroduce exactly the
// silent-corruption risk the truncation check exists to remove.
export const isRetryable = (err: unknown): boolean => {
    if (err instanceof ApiError) {
        // 408 and 429 are the two 4xx codes that are statements about timing
        // rather than about the request, and backoff is the right answer to
        // both. Every other 4xx will answer the same way however often it is
        // asked.
        if (err.status === 408 || err.status === 429) return true;
        return err.status >= 500 && err.status <= 599;
    }
    if (err instanceof TruncatedStreamError) return true;
    // quak only ever aborts a request on its own deadline, so an abort means
    // this attempt ran out of time — which a later one may not.
    if (isAbort(err)) return true;
    // Node's fetch rejects with `TypeError: fetch failed` for everything below
    // HTTP: DNS failure, refused connection, TLS error, reset socket. Nothing
    // on the object separates it from a TypeError thrown by a bug, so this is
    // deliberately literal. Demanding a recognised `cause` instead would
    // classify real network failures as permanent and fail backups that should
    // have succeeded; the cost of the imprecision is bounded by the attempt
    // count. Bun's fetch rejects with a plain `Error` instead, which is
    // recognised by its code (see `TRANSPORT_CODES`).
    if (err instanceof TypeError) return true;
    return hasTransportCode(err);
};

// Could the first attempt already have taken effect on the server?
//
// `isRetryable` is the wrong question for a request that changes state.
// `postJSON` and `putJSON` use this for every `POST` and `PUT` listed in the
// README under "Endpoints used"; verifying a second factor, for one, consumes
// one of a small number of attempts. They are replayed only on the failures in
// `CONNECT_CODES`, which establish that no TCP connection to the server ever
// existed: there was no address to connect to, or the peer refused the
// connection outright. A request byte cannot have been transmitted, so the
// server cannot have acted.
//
// Everything else is ambiguous. A 5xx proves the server did process the
// request. A reset or a broken pipe can arrive after it was fully sent and
// acted on. A routing errno can be delivered on an established socket. Bun's
// `ConnectionRefused` and `FailedToOpenSocket` can come from a new connection
// bun opened to resend a request that already went out on a connection it kept
// open. A deadline says nothing at all about the server's state. So every code in the
// cause chain must be one of `CONNECT_CODES`: one other code anywhere in the
// chain is doubt, and doubt is not replayed. A chain longer than the walk is
// doubt too: the links below the limit were never read.
export const isSafeToReplay = (err: unknown): boolean => {
    const { codes, complete } = causeCodes(err);
    return (
        isRetryable(err) &&
        complete &&
        codes.length > 0 &&
        codes.every((code) => CONNECT_CODES.has(code))
    );
};

export interface WithRetryOptions extends RetryOptions {
    isRetryable?: (err: unknown) => boolean;
}

// Exponential backoff with full jitter: the exponential term is the ceiling,
// and the actual wait is drawn uniformly below it. Full jitter, rather than a
// fixed delay plus noise, is what stops a client that lost a hundred parallel
// downloads to one CDN blip from re-sending all hundred at the same instant.
const backoffMs = (retryNumber: number, policy: ResolvedRetryOptions): number =>
    policy.random() *
    Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retryNumber - 1));

export const withRetry = async <T>(
    fn: () => Promise<T>,
    opts?: WithRetryOptions,
): Promise<T> => {
    const policy = resolveRetryOptions(opts);
    const retryable = opts?.isRetryable ?? isRetryable;
    for (let attempt = 1; ; attempt++) {
        try {
            return await fn();
        } catch (err) {
            if (attempt >= policy.attempts || !retryable(err)) {
                // The error escapes unwrapped, and it is the one from the
                // final attempt: callers classify what they catch, and
                // `listMissingThumbnails` in particular needs the `ApiError`
                // and its status intact.
                throw err;
            }
            await policy.sleep(backoffMs(attempt, policy));
        }
    }
};