sneak
e046d696cd
check / check (push) Successful in 16s
downloadFile, shared by quak get, the content cache and backup, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and stores nothing on a mismatch, failing with an error naming the file ID. A live photo ZIP is unpacked as it streams with fflate's Unzip, in small slices so memory stays bounded however far an entry expands, and its image and video hashed separately as <imageHash>:<videoHash>. decryptFile reads the older imageHash/videoHash fields for live photos. A file with no recorded hash is stored unchecked. Model: opus-5-5
1751 lines
73 KiB
TypeScript
1751 lines
73 KiB
TypeScript
/**
|
|
* Tests for `downloadFile` and `downloadThumbnail`.
|
|
*
|
|
* These functions combine the ApiClient's streaming download with the
|
|
* secretstream pull decryption to recover the plaintext file content and
|
|
* write it to disk.
|
|
*
|
|
* The encrypted body returned by the CDN is a concatenation of
|
|
* secretstream ciphertext chunks. Each chunk is at most
|
|
* `STREAM_CHUNK_SIZE + STREAM_CHUNK_OVERHEAD` bytes (4 MiB + 17 bytes).
|
|
* The download function buffers incoming network data, splits it on the
|
|
* chunk boundary, and feeds each piece to `pullStreamChunk`.
|
|
*
|
|
* Two contracts are load-bearing for anyone using this library as a backup
|
|
* tool, and both are documented by the tests below:
|
|
*
|
|
* 1. **Truncation is an error, never a short file.** Only the final chunk of
|
|
* a secretstream carries `TAG_FINAL`. A download cut short by a dropped
|
|
* connection still decrypts cleanly up to the last whole chunk, so without
|
|
* an explicit `TAG_FINAL` check a truncated body is indistinguishable from
|
|
* a complete one. `streamDecrypt` therefore refuses to return unless the
|
|
* stream ended on `TAG_FINAL`, and the error says the stream was truncated.
|
|
* A transfer that stopped part-way through a chunk is reported the same
|
|
* way, since a final chunk that arrived in full always authenticates.
|
|
*
|
|
* 2. **The destination path is written atomically.** Plaintext goes to a
|
|
* temporary sibling file first and is `rename`d into place only after the
|
|
* whole stream has decrypted and verified. A caller that sees no exception
|
|
* can rely on the destination containing the complete, authenticated file;
|
|
* a caller that sees an exception can rely on the destination being
|
|
* untouched — whatever was there before is still there, byte for byte, and
|
|
* no partial file has appeared. This matters because `runBackup` skips any
|
|
* existing non-empty file, so a partial write would be treated as complete
|
|
* forever after. The staging file and the rename are observed directly (see
|
|
* the `rename` hook below), not inferred from an empty directory.
|
|
*
|
|
* 3. **A failed transfer is retried as a whole.** A download is a request, a
|
|
* stream consumption, and a decryption, and only the first of those three
|
|
* happens inside `ApiClient`. A socket reset after the response headers
|
|
* have arrived therefore surfaces here, in the download layer — and that is
|
|
* the dominant failure mode for multi-megabyte photos over a CDN. So the
|
|
* entire sequence is retried as one unit, not just the request. The
|
|
* secretstream pull state is not resumable and there is no Range support,
|
|
* so a retry starts the file over from byte zero.
|
|
*
|
|
* These tests build synthetic encrypted files using sodium's push API,
|
|
* serve them from a mock fetch, and verify the decrypted output on disk.
|
|
*/
|
|
|
|
import {
|
|
chmodSync,
|
|
existsSync,
|
|
mkdirSync,
|
|
readdirSync,
|
|
readFileSync,
|
|
rmSync,
|
|
mkdtempSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { createHash } from "node:crypto";
|
|
import sodium from "libsodium-wrappers-sumo";
|
|
import { zipSync } from "fflate";
|
|
import {
|
|
beforeAll,
|
|
beforeEach,
|
|
afterAll,
|
|
describe,
|
|
expect,
|
|
it,
|
|
vi,
|
|
} from "vitest";
|
|
import { init, toBase64, STREAM_CHUNK_SIZE } from "../../src/crypto/index.js";
|
|
import { ApiClient } from "../../src/api/client.js";
|
|
import { ApiError, TruncatedStreamError } from "../../src/errors.js";
|
|
import type { RetryOptions } from "../../src/retry.js";
|
|
import {
|
|
downloadFile,
|
|
downloadThumbnail,
|
|
writeAtomic,
|
|
} from "../../src/download/index.js";
|
|
import type { EnteFile, FileMetadata } from "../../src/model/types.js";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* `rename` is intercepted so that the tests can observe — and fail — the
|
|
* final step of the atomic write.
|
|
*
|
|
* Every other failure in this file is injected inside `streamDecrypt`, which
|
|
* runs before anything is written to disk. Those tests therefore cannot tell
|
|
* an atomic write from a plain `writeFile`: under both, nothing was ever
|
|
* created, so an empty directory proves nothing about cleanup. This hook is
|
|
* what closes that gap. It records each rename the downloader performs,
|
|
* including whether the source existed at that moment (i.e. that the staged
|
|
* temp file really was written), and can be told to fail the rename so the
|
|
* cleanup path runs with a temp file genuinely on disk.
|
|
*
|
|
* `vi.hoisted` is required: `vi.mock` factories are hoisted above the imports,
|
|
* so a plain module-level `const` would still be in its temporal dead zone by
|
|
* the time the factory runs.
|
|
*/
|
|
const renameHook = vi.hoisted(() => ({
|
|
calls: [] as { from: string; to: string; sourceExisted: boolean }[],
|
|
failWith: null as Error | null,
|
|
}));
|
|
|
|
/**
|
|
* `open` is wrapped so the tests can observe the durability fsyncs the atomic
|
|
* writer performs — which are otherwise invisible: an fsync leaves no trace in
|
|
* the file's contents. Each `FileHandle.sync()` is recorded, and rename and
|
|
* sync events are appended to a single ordered `events` log so a test can pin
|
|
* the sequence "fsync the temp file, rename, fsync the directory" that makes a
|
|
* write survive a power cut. The flag the handle was opened with distinguishes
|
|
* the temp file (`w`) from its containing directory (`r`).
|
|
*/
|
|
const durabilityHook = vi.hoisted(() => ({
|
|
events: [] as string[],
|
|
}));
|
|
|
|
/**
|
|
* `FileHandle.write` is wrapped so the tests can watch the streaming decrypt
|
|
* path put plaintext on disk one chunk at a time. This is the direct evidence
|
|
* that memory is bounded by the chunk size and not the file size: a buffered
|
|
* downloader would hand the whole file to a single write, whereas the streaming
|
|
* one issues one write per secretstream chunk, none larger than
|
|
* `STREAM_CHUNK_SIZE`. Each write records the temp path it targeted and its
|
|
* length. `writeFile` (which the whole-buffer `writeAtomic` uses) is a distinct
|
|
* native call and does not go through this method, so only the streaming path
|
|
* is observed here.
|
|
*/
|
|
const writeHook = vi.hoisted(() => ({
|
|
writes: [] as { path: string; length: number }[],
|
|
}));
|
|
|
|
vi.mock("node:fs/promises", async (importOriginal) => {
|
|
const actual = await importOriginal<typeof import("node:fs/promises")>();
|
|
const { existsSync: sourceExists } = await import("node:fs");
|
|
return {
|
|
...actual,
|
|
open: async (
|
|
path: Parameters<typeof actual.open>[0],
|
|
flags?: Parameters<typeof actual.open>[1],
|
|
...rest: unknown[]
|
|
): Promise<Awaited<ReturnType<typeof actual.open>>> => {
|
|
const handle = await actual.open(
|
|
path,
|
|
flags as Parameters<typeof actual.open>[1],
|
|
...(rest as []),
|
|
);
|
|
const realSync = handle.sync.bind(handle);
|
|
handle.sync = async (): Promise<void> => {
|
|
durabilityHook.events.push(`sync:${String(flags)}:${path}`);
|
|
await realSync();
|
|
};
|
|
const realWrite = handle.write.bind(handle);
|
|
handle.write = (async (
|
|
data: unknown,
|
|
...rest2: unknown[]
|
|
): Promise<unknown> => {
|
|
if (data instanceof Uint8Array) {
|
|
writeHook.writes.push({
|
|
path: String(path),
|
|
length: data.length,
|
|
});
|
|
}
|
|
return (realWrite as (...a: unknown[]) => Promise<unknown>)(
|
|
data,
|
|
...rest2,
|
|
);
|
|
}) as typeof handle.write;
|
|
return handle;
|
|
},
|
|
rename: async (from: string, to: string): Promise<void> => {
|
|
renameHook.calls.push({
|
|
from,
|
|
to,
|
|
sourceExisted: sourceExists(from),
|
|
});
|
|
durabilityHook.events.push(`rename:${to}`);
|
|
if (renameHook.failWith !== null) {
|
|
throw renameHook.failWith;
|
|
}
|
|
await actual.rename(from, to);
|
|
},
|
|
};
|
|
});
|
|
|
|
/**
|
|
* `chunkHashUpdate` is wrapped to record the length of every piece hashed, so
|
|
* a test can show that a live photo entry reaches the hash in pieces far
|
|
* smaller than the entry, rather than decompressed whole first.
|
|
*/
|
|
const hashHook = vi.hoisted(() => ({
|
|
lengths: [] as number[],
|
|
}));
|
|
|
|
vi.mock("../../src/crypto/index.js", async (importOriginal) => {
|
|
const actual =
|
|
await importOriginal<typeof import("../../src/crypto/index.js")>();
|
|
return {
|
|
...actual,
|
|
chunkHashUpdate: (
|
|
...args: Parameters<typeof actual.chunkHashUpdate>
|
|
): void => {
|
|
hashHook.lengths.push(args[1].length);
|
|
actual.chunkHashUpdate(...args);
|
|
},
|
|
};
|
|
});
|
|
|
|
beforeEach(() => {
|
|
hashHook.lengths.length = 0;
|
|
renameHook.calls.length = 0;
|
|
renameHook.failWith = null;
|
|
durabilityHook.events.length = 0;
|
|
writeHook.writes.length = 0;
|
|
});
|
|
|
|
let testDir: string;
|
|
|
|
beforeAll(async () => {
|
|
await init();
|
|
await sodium.ready;
|
|
testDir = mkdtempSync(join(tmpdir(), "quak-test-"));
|
|
});
|
|
|
|
afterAll(() => {
|
|
if (testDir && existsSync(testDir)) {
|
|
rmSync(testDir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
/**
|
|
* Deterministic stand-in for random test payloads.
|
|
*
|
|
* Fixture *content* is never load-bearing here — the assertions turn on
|
|
* length, framing, and the secretstream tag — but it must not be a constant
|
|
* fill either, or a downloader that reordered or repeated chunks would still
|
|
* produce the expected bytes. A seeded linear congruential generator gives
|
|
* both: byte patterns that differ across every offset and seed, reproducible
|
|
* on any machine, which is what the README asks of fixtures.
|
|
*
|
|
* It is also the difference between a fast suite and a broken one.
|
|
* `sodium.randombytes_buf` goes through the wasm wrapper a byte at a time and
|
|
* costs roughly 20 seconds for the 4 MiB chunk below — about two hundred
|
|
* times what it costs to encrypt the same buffer, and on its own enough to
|
|
* push `make test` past the 30-second cap in `script/test`. This loop fills
|
|
* 4 MiB in a few milliseconds.
|
|
*/
|
|
const patternBytes = (length: number, seed: number): Uint8Array => {
|
|
const out = new Uint8Array(length);
|
|
let x = seed >>> 0;
|
|
for (let i = 0; i < length; i++) {
|
|
// Numerical Recipes' LCG constants; the high byte is used because
|
|
// the low bits of an LCG have short periods.
|
|
x = (Math.imul(x, 1664525) + 1013904223) >>> 0;
|
|
out[i] = (x >>> 24) & 0xff;
|
|
}
|
|
return out;
|
|
};
|
|
|
|
/**
|
|
* Encrypt `plaintext` as a secretstream file body (single chunk with
|
|
* TAG_FINAL). Returns the key, header, and ciphertext that the mock CDN
|
|
* will serve.
|
|
*/
|
|
const encryptFileBody = (
|
|
plaintext: Uint8Array,
|
|
key: Uint8Array,
|
|
): { header: Uint8Array; ciphertext: Uint8Array } => {
|
|
const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
|
const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push(
|
|
push.state,
|
|
plaintext,
|
|
null,
|
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
|
);
|
|
return { header: push.header, ciphertext };
|
|
};
|
|
|
|
/**
|
|
* Encrypt a body that spans more than one secretstream chunk, the way the
|
|
* server does for files larger than the 4 MiB plaintext chunk size.
|
|
*
|
|
* Framing matters here: the downloader splits the byte stream on fixed
|
|
* `STREAM_CHUNK_SIZE + STREAM_CHUNK_OVERHEAD` boundaries, so every chunk
|
|
* except the last must carry exactly `STREAM_CHUNK_SIZE` plaintext bytes.
|
|
* Only the last chunk is tagged `TAG_FINAL`; the leading ones are
|
|
* `TAG_MESSAGE`.
|
|
*
|
|
* Returns the header, the concatenated body, the plaintext it decrypts to,
|
|
* and `finalChunkOffset` — the byte offset at which the `TAG_FINAL` chunk
|
|
* begins, so a test can slice it off to simulate a connection that dropped
|
|
* before the end of the file.
|
|
*/
|
|
const encryptMultiChunkBody = (
|
|
key: Uint8Array,
|
|
leadingChunks: number,
|
|
finalChunkPlainSize: number,
|
|
): {
|
|
header: Uint8Array;
|
|
body: Uint8Array;
|
|
plaintext: Uint8Array;
|
|
finalChunkOffset: number;
|
|
} => {
|
|
const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
|
const cipherParts: Uint8Array[] = [];
|
|
const plainParts: Uint8Array[] = [];
|
|
|
|
for (let i = 0; i < leadingChunks; i++) {
|
|
const plain = patternBytes(STREAM_CHUNK_SIZE, i + 1);
|
|
plainParts.push(plain);
|
|
cipherParts.push(
|
|
sodium.crypto_secretstream_xchacha20poly1305_push(
|
|
push.state,
|
|
plain,
|
|
null,
|
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE,
|
|
),
|
|
);
|
|
}
|
|
|
|
const finalPlain = patternBytes(finalChunkPlainSize, leadingChunks + 1);
|
|
plainParts.push(finalPlain);
|
|
const finalCipher = sodium.crypto_secretstream_xchacha20poly1305_push(
|
|
push.state,
|
|
finalPlain,
|
|
null,
|
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
|
);
|
|
|
|
const finalChunkOffset = cipherParts.reduce((n, c) => n + c.length, 0);
|
|
cipherParts.push(finalCipher);
|
|
|
|
return {
|
|
header: push.header,
|
|
body: concat(cipherParts),
|
|
plaintext: concat(plainParts),
|
|
finalChunkOffset,
|
|
};
|
|
};
|
|
|
|
const concat = (parts: Uint8Array[]): Uint8Array => {
|
|
const total = parts.reduce((n, p) => n + p.length, 0);
|
|
const out = new Uint8Array(total);
|
|
let offset = 0;
|
|
for (const p of parts) {
|
|
out.set(p, offset);
|
|
offset += p.length;
|
|
}
|
|
return out;
|
|
};
|
|
|
|
const buildMockEnteFile = (
|
|
key: Uint8Array,
|
|
fileHeader: Uint8Array,
|
|
thumbHeader: Uint8Array,
|
|
): EnteFile => ({
|
|
id: 999,
|
|
collectionID: 1,
|
|
ownerID: 1,
|
|
key,
|
|
metadata: {
|
|
title: "test-photo.jpg",
|
|
fileType: "image",
|
|
creationTime: 0,
|
|
modificationTime: 0,
|
|
} as FileMetadata,
|
|
file: { decryptionHeader: toBase64(fileHeader) },
|
|
thumbnail: { decryptionHeader: toBase64(thumbHeader) },
|
|
updationTime: 0,
|
|
});
|
|
|
|
const mockFetchForBody = (body: Uint8Array) => {
|
|
const fake = async (): Promise<Response> =>
|
|
new Response(body, { status: 200 });
|
|
return fake as typeof globalThis.fetch;
|
|
};
|
|
|
|
/**
|
|
* Encrypt a body consisting of one chunk that is *not* tagged TAG_FINAL.
|
|
*
|
|
* This is the cheap way to present a stream that ended without its final
|
|
* chunk: the downloader pulls it, authenticates it, and finds the stream
|
|
* over on a TAG_MESSAGE chunk — the same terminal condition as a large file
|
|
* whose last chunk was lost, without paying for a 4 MiB fixture. The
|
|
* multi-chunk fixture above covers the realistic wire shape; this one is
|
|
* used where the test is really about what happens on disk afterwards.
|
|
*/
|
|
const encryptNonFinalBody = (
|
|
plaintext: Uint8Array,
|
|
key: Uint8Array,
|
|
): { header: Uint8Array; ciphertext: Uint8Array } => {
|
|
const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
|
const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push(
|
|
push.state,
|
|
plaintext,
|
|
null,
|
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE,
|
|
);
|
|
return { header: push.header, ciphertext };
|
|
};
|
|
|
|
/**
|
|
* Compare file contents by digest rather than with `toEqual`. Vitest's deep
|
|
* equality walks multi-megabyte buffers byte by byte, which costs seconds on
|
|
* the 4 MiB fixtures; a digest comparison is exact and effectively free.
|
|
*/
|
|
const expectSameBytes = (actual: Uint8Array, expected: Uint8Array): void => {
|
|
expect(actual.length).toBe(expected.length);
|
|
expect(createHash("sha256").update(actual).digest("hex")).toBe(
|
|
createHash("sha256").update(expected).digest("hex"),
|
|
);
|
|
};
|
|
|
|
/**
|
|
* A multi-chunk fixture shared by the truncation tests: one full 4 MiB
|
|
* `TAG_MESSAGE` chunk followed by a small `TAG_FINAL` chunk. It is built once
|
|
* and shared because encrypting 4 MiB costs about 100ms. Its plaintext is
|
|
* generated rather than drawn from the CSPRNG, which is what keeps that
|
|
* encryption the whole cost of the fixture.
|
|
*/
|
|
let multiChunk: ReturnType<typeof encryptMultiChunkBody>;
|
|
let multiChunkKey: Uint8Array;
|
|
|
|
beforeAll(() => {
|
|
multiChunkKey = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
multiChunk = encryptMultiChunkBody(multiChunkKey, 1, 1024);
|
|
});
|
|
|
|
/** An error shaped like a Node transport failure: the errno is on `.code`. */
|
|
const errnoError = (code: string, message = code): Error =>
|
|
Object.assign(new Error(message), { code });
|
|
|
|
/**
|
|
* A retry policy with the waiting removed, used by every fixture in this
|
|
* file. Backoff arithmetic belongs to `test/retry/retry.test.ts`; here the
|
|
* only interesting quantity is how many requests a download issued, so the
|
|
* injected `sleep` returns immediately and nothing in this file waits.
|
|
*/
|
|
const noWait: RetryOptions = {
|
|
sleep: () => Promise.resolve(),
|
|
random: () => 0,
|
|
};
|
|
|
|
/**
|
|
* One scripted outcome for a single request to the CDN.
|
|
*
|
|
* - `body` — a complete response body.
|
|
* - `status` — an HTTP error response.
|
|
* - `reset` — a response whose headers arrive, whose body delivers `bytes`,
|
|
* and which then dies with a socket reset. This is the failure that
|
|
* motivates retrying the download rather than the request: by the time it
|
|
* happens `ApiClient` has already returned successfully.
|
|
*/
|
|
type BodyStep =
|
|
| { kind: "body"; bytes: Uint8Array }
|
|
| { kind: "status"; status: number }
|
|
| { kind: "reset"; bytes: Uint8Array };
|
|
|
|
/**
|
|
* A fetch that serves one scripted step per call and counts the calls. It
|
|
* deliberately refuses to serve more requests than it was given steps for, so
|
|
* a retry loop that ran away is a test failure rather than a silent success.
|
|
*/
|
|
const scriptedCdnFetch = (
|
|
...steps: BodyStep[]
|
|
): { fetch: typeof globalThis.fetch; requests: () => number } => {
|
|
let calls = 0;
|
|
const fake = async (): Promise<Response> => {
|
|
const step = steps[calls++];
|
|
if (step === undefined) {
|
|
throw new Error(`scriptedCdnFetch: no step for request #${calls}`);
|
|
}
|
|
if (step.kind === "status") {
|
|
return new Response("error", { status: step.status });
|
|
}
|
|
if (step.kind === "body") {
|
|
return new Response(step.bytes, { status: 200 });
|
|
}
|
|
const bytes = step.bytes;
|
|
return new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(bytes);
|
|
controller.error(
|
|
errnoError("ECONNRESET", "aborted by peer"),
|
|
);
|
|
},
|
|
}),
|
|
{ status: 200 },
|
|
);
|
|
};
|
|
return { fetch: fake as typeof globalThis.fetch, requests: () => calls };
|
|
};
|
|
|
|
/**
|
|
* Build an EnteFile plus ApiClient whose file *and* thumbnail streams both
|
|
* serve `body` under `header`. The download path under test is otherwise
|
|
* identical for the two, so every truncation/atomicity case below runs
|
|
* against both entry points from a single fixture.
|
|
*
|
|
* The default policy here is a single attempt. The failure-contract tests are
|
|
* about what the caller and the filesystem are left with, not about how many
|
|
* times quak asked; pinning attempts to one keeps them saying exactly that,
|
|
* and keeps them from re-decrypting a 4 MiB fixture four times over. The
|
|
* retry counts have their own tests at the bottom of this file, which set the
|
|
* attempt count explicitly.
|
|
*/
|
|
const fixtureFor = (
|
|
key: Uint8Array,
|
|
header: Uint8Array,
|
|
body: Uint8Array,
|
|
retry: RetryOptions = { ...noWait, attempts: 1 },
|
|
): { api: ApiClient; file: EnteFile } => ({
|
|
api: new ApiClient({ fetch: mockFetchForBody(body), retry }),
|
|
file: buildMockEnteFile(key, header, header),
|
|
});
|
|
|
|
// The two entry points share `streamDecrypt` and the atomic-write wrapper,
|
|
// so the contract tests are written once and run against both.
|
|
const entryPoints = [
|
|
{ name: "downloadFile", download: downloadFile },
|
|
{ name: "downloadThumbnail", download: downloadThumbnail },
|
|
];
|
|
|
|
// With no `outPath`, the destination is named after `metadata.title`, relative
|
|
// to the working directory. Such tests run inside a temporary directory:
|
|
// `make check` must not create files in the repo root.
|
|
const inDirectory = async <T>(
|
|
dir: string,
|
|
run: () => Promise<T>,
|
|
): Promise<T> => {
|
|
const previous = process.cwd();
|
|
process.chdir(dir);
|
|
try {
|
|
return await run();
|
|
} finally {
|
|
process.chdir(previous);
|
|
}
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe("downloadFile", () => {
|
|
it("downloads, decrypts, and writes a single-chunk file", async () => {
|
|
const plaintext = new TextEncoder().encode(
|
|
"Hello from quak! This is a test photo payload.",
|
|
);
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
|
|
|
// Separate header for thumbnail (not used in this test path but
|
|
// needed to construct the EnteFile)
|
|
const thumbPush =
|
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
|
|
|
const file = buildMockEnteFile(key, header, thumbPush.header);
|
|
const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) });
|
|
const outPath = join(testDir, "single-chunk.jpg");
|
|
|
|
const result = await downloadFile(api, file, outPath);
|
|
|
|
// The whole DownloadResult shape is asserted, not just its fields:
|
|
// callers depend on `path` being the destination they asked for
|
|
// (never the temporary file used along the way) and on
|
|
// `bytesWritten` being the plaintext length.
|
|
expect(result).toEqual({
|
|
path: outPath,
|
|
bytesWritten: plaintext.length,
|
|
});
|
|
expect(readFileSync(outPath)).toEqual(Buffer.from(plaintext));
|
|
});
|
|
|
|
it("uses metadata.title as filename when outPath is omitted", async () => {
|
|
const plaintext = new Uint8Array([1, 2, 3]);
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
|
const thumbPush =
|
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
|
const file = buildMockEnteFile(key, header, thumbPush.header);
|
|
file.metadata.title = "fallback-name.png";
|
|
const dir = mkdtempSync(join(testDir, "title-"));
|
|
|
|
const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) });
|
|
const result = await inDirectory(dir, () => downloadFile(api, file));
|
|
|
|
expect(result.path).toBe("fallback-name.png");
|
|
expect(readFileSync(join(dir, "fallback-name.png"))).toEqual(
|
|
Buffer.from(plaintext),
|
|
);
|
|
});
|
|
|
|
it("keeps a hostile title inside the working directory", async () => {
|
|
// The server controls the title. `../escaped.png` must not write to
|
|
// the parent directory; it becomes one file name in the current one.
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.body,
|
|
);
|
|
file.metadata.title = "../escaped.png";
|
|
const parent = mkdtempSync(join(testDir, "hostile-"));
|
|
const dir = join(parent, "cwd");
|
|
mkdirSync(dir);
|
|
|
|
const result = await inDirectory(dir, () => downloadFile(api, file));
|
|
|
|
expect(result.path).toBe("__escaped.png");
|
|
expect(readdirSync(dir)).toEqual(["__escaped.png"]);
|
|
expect(readdirSync(parent)).toEqual(["cwd"]);
|
|
});
|
|
|
|
it("uses an explicit outPath verbatim, even one with ..", async () => {
|
|
// The caller is trusted: its path is not sanitized.
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.body,
|
|
);
|
|
const dir = mkdtempSync(join(testDir, "explicit-"));
|
|
mkdirSync(join(dir, "sub"));
|
|
const outPath = join(dir, "sub", "..", "explicit.bin");
|
|
|
|
const result = await downloadFile(api, file, outPath);
|
|
|
|
expect(result.path).toBe(outPath);
|
|
expect(existsSync(join(dir, "explicit.bin"))).toBe(true);
|
|
});
|
|
|
|
it("handles a larger single-chunk file (random binary payload)", async () => {
|
|
// Most photos are under 4 MiB and therefore a single secretstream
|
|
// chunk. This test exercises a non-trivial payload size with
|
|
// arbitrary binary data (not just ASCII) to verify no encoding bugs.
|
|
const plaintext = patternBytes(100_000, 11);
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
|
const thumbPush =
|
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
|
const file = buildMockEnteFile(key, header, thumbPush.header);
|
|
|
|
const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) });
|
|
const outPath = join(testDir, "large-single.bin");
|
|
const result = await downloadFile(api, file, outPath);
|
|
|
|
expect(result.bytesWritten).toBe(100_000);
|
|
expectSameBytes(readFileSync(outPath), plaintext);
|
|
});
|
|
|
|
it("decrypts a body that spans several secretstream chunks", async () => {
|
|
// Files over 4 MiB arrive as several ciphertext chunks concatenated
|
|
// into one HTTP body. The downloader has to re-split them on the
|
|
// exact chunk boundary; getting that wrong corrupts every large
|
|
// photo in an account. This is also the positive control for the
|
|
// truncation tests below: it proves the multi-chunk fixture itself
|
|
// decrypts cleanly when nothing has been removed from it.
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.body,
|
|
);
|
|
const outPath = join(testDir, "multi-chunk.bin");
|
|
|
|
const result = await downloadFile(api, file, outPath);
|
|
|
|
expect(result.bytesWritten).toBe(multiChunk.plaintext.length);
|
|
expectSameBytes(readFileSync(outPath), multiChunk.plaintext);
|
|
});
|
|
});
|
|
|
|
describe("downloadThumbnail", () => {
|
|
it("downloads and decrypts the thumbnail stream", async () => {
|
|
const plaintext = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]); // JPEG SOI
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
|
|
const filePush =
|
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
|
const { header: thumbHeader, ciphertext: thumbCipher } =
|
|
encryptFileBody(plaintext, key);
|
|
|
|
const file = buildMockEnteFile(key, filePush.header, thumbHeader);
|
|
const api = new ApiClient({ fetch: mockFetchForBody(thumbCipher) });
|
|
const outPath = join(testDir, "thumb.jpg");
|
|
|
|
const result = await downloadThumbnail(api, file, outPath);
|
|
|
|
expect(result).toEqual({ path: outPath, bytesWritten: 4 });
|
|
expect(readFileSync(outPath)).toEqual(Buffer.from(plaintext));
|
|
});
|
|
|
|
it("names the thumbnail thumb_ plus the sanitized title", async () => {
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.body,
|
|
);
|
|
file.metadata.title = "/etc/passwd";
|
|
const dir = mkdtempSync(join(testDir, "thumb-title-"));
|
|
|
|
const result = await inDirectory(dir, () =>
|
|
downloadThumbnail(api, file),
|
|
);
|
|
|
|
expect(result.path).toBe("thumb__etc_passwd");
|
|
expect(readdirSync(dir)).toEqual(["thumb__etc_passwd"]);
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Truncation detection and atomic writes
|
|
//
|
|
// Everything below is the failure contract. It is deliberately written once
|
|
// per entry point via `entryPoints`, because `downloadFile` and
|
|
// `downloadThumbnail` must behave identically here: a corrupt thumbnail is
|
|
// just as unacceptable as a corrupt original, and `runBackup` trusts both.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe.each(entryPoints)(
|
|
"$name truncation handling",
|
|
({ name, download }) => {
|
|
/** A fresh, empty directory so leftover-file assertions are meaningful. */
|
|
const freshDir = (): string => {
|
|
const dir = mkdtempSync(join(testDir, `${name}-`));
|
|
return dir;
|
|
};
|
|
|
|
it("rejects a body whose final TAG_FINAL chunk never arrived", async () => {
|
|
// Simulate a connection that dropped after the first 4 MiB chunk.
|
|
// Every byte that did arrive decrypts and authenticates perfectly —
|
|
// that is precisely the danger. The only signal that the file is
|
|
// incomplete is the absence of a chunk tagged TAG_FINAL, so the
|
|
// downloader must treat "stream ended on TAG_MESSAGE" as a hard
|
|
// error rather than returning a short file.
|
|
const truncatedBody = multiChunk.body.slice(
|
|
0,
|
|
multiChunk.finalChunkOffset,
|
|
);
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
truncatedBody,
|
|
);
|
|
const outPath = join(freshDir(), "truncated.bin");
|
|
|
|
const err: unknown = await download(api, file, outPath).catch(
|
|
(e: unknown) => e,
|
|
);
|
|
|
|
// The type, not the wording, is the contract. The retry policy
|
|
// classifies truncation as worth another attempt, and it decides
|
|
// that with `instanceof`: matching on message text would make
|
|
// rewording a diagnostic silently turn every truncated download
|
|
// into a permanent failure.
|
|
expect(err).toBeInstanceOf(TruncatedStreamError);
|
|
expect((err as Error).message).toMatch(/truncated/i);
|
|
});
|
|
|
|
it("rejects a body whose final chunk arrived only in part", async () => {
|
|
// The likelier shape of a dropped connection: the transfer stops
|
|
// in the middle of a chunk rather than neatly between two. The
|
|
// bytes that arrived are a prefix of a complete chunk, so Poly1305
|
|
// rejects them — which is, cryptographically, indistinguishable
|
|
// from corruption of a whole chunk.
|
|
//
|
|
// It is still reported as truncation, because that is what it
|
|
// almost always is and because this library's entire reason for
|
|
// checking TAG_FINAL is to make a short transfer visible. Calling
|
|
// a short transfer "authentication failed" would send a user
|
|
// hunting for a corrupt file when their network is at fault. The
|
|
// underlying authentication failure is kept as the error's
|
|
// `cause`, so the real diagnosis is never lost.
|
|
const shortBody = multiChunk.body.slice(
|
|
0,
|
|
multiChunk.body.length - 8,
|
|
);
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
shortBody,
|
|
);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "partial-final.bin");
|
|
|
|
const err = await download(api, file, outPath).catch(
|
|
(e: unknown) => e,
|
|
);
|
|
|
|
expect(err).toBeInstanceOf(TruncatedStreamError);
|
|
expect((err as Error).message).toMatch(/truncated/i);
|
|
expect((err as Error).cause).toBeInstanceOf(Error);
|
|
expect(((err as Error).cause as Error).message).toMatch(
|
|
/authentication failed/i,
|
|
);
|
|
|
|
// And, as with every other failure, the destination is untouched
|
|
// and no staged temp file survives.
|
|
expect(existsSync(outPath)).toBe(false);
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
});
|
|
|
|
it("rejects an empty body instead of writing a zero-byte file", async () => {
|
|
// Ente always emits at least one chunk, even for empty content:
|
|
// `encryptBlob` shows that a zero-length plaintext still produces a
|
|
// TAG_FINAL chunk. A body with no chunks at all therefore means the
|
|
// transfer failed, not that the file is empty. Writing a zero-byte
|
|
// file here would be the worst outcome, because `runBackup` would
|
|
// then see a file it considers present and never retry it.
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
new Uint8Array(0),
|
|
);
|
|
const outPath = join(freshDir(), "empty.bin");
|
|
|
|
await expect(download(api, file, outPath)).rejects.toBeInstanceOf(
|
|
TruncatedStreamError,
|
|
);
|
|
});
|
|
|
|
it("leaves no file at the destination after a truncated download", async () => {
|
|
// The caller's contract: if the promise rejects, the destination
|
|
// path does not exist. Nothing downstream should ever have to guess
|
|
// whether a leftover file is complete.
|
|
//
|
|
// The body here is a single chunk that was never tagged TAG_FINAL,
|
|
// which puts the downloader in exactly the terminal state a lost
|
|
// last chunk produces, without the cost of a 4 MiB fixture. What
|
|
// this test is really about is the state of the filesystem after
|
|
// the rejection.
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptNonFinalBody(
|
|
patternBytes(256, 21),
|
|
key,
|
|
);
|
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "absent.bin");
|
|
|
|
await expect(download(api, file, outPath)).rejects.toBeInstanceOf(
|
|
TruncatedStreamError,
|
|
);
|
|
|
|
expect(existsSync(outPath)).toBe(false);
|
|
// And no temporary scratch file is left behind either: the download
|
|
// stages plaintext in a sibling temp file, which must be removed on
|
|
// the failure path so repeated failures cannot fill the disk.
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
});
|
|
|
|
it("reports a corrupt whole chunk as an authentication failure, not truncation", async () => {
|
|
// The counterpart to the partial-final-chunk case above, and the
|
|
// reason the two are distinguishable at all. A byte is flipped
|
|
// inside the first chunk of a multi-chunk body: that chunk arrives
|
|
// complete — the stream goes on past it — so its failure cannot be
|
|
// a short transfer. It is corruption, and the caller is told so,
|
|
// with `pullStreamChunk`'s error propagated unchanged because it is
|
|
// the real diagnosis.
|
|
//
|
|
// The same on-disk guarantee holds for this failure mode as for
|
|
// every other: nothing at the destination, nothing left over.
|
|
const corrupted = Uint8Array.from(multiChunk.body);
|
|
corrupted[10] ^= 0xff;
|
|
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
corrupted,
|
|
);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "corrupt.bin");
|
|
|
|
const err: unknown = await download(api, file, outPath).catch(
|
|
(e: unknown) => e,
|
|
);
|
|
|
|
expect(err).toBeInstanceOf(Error);
|
|
expect((err as Error).message).toMatch(/authentication failed/i);
|
|
// And explicitly *not* the truncation type, because that type is
|
|
// what the retry policy keys on: mislabelling corruption as
|
|
// truncation would spend the whole attempt budget re-downloading
|
|
// a file that will never decrypt.
|
|
expect(err).not.toBeInstanceOf(TruncatedStreamError);
|
|
|
|
expect(existsSync(outPath)).toBe(false);
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
});
|
|
|
|
it("does not clobber an existing file when the download fails", async () => {
|
|
// The repair case. A user re-running a backup over a directory that
|
|
// already holds good originals must never end up worse off: a failed
|
|
// download leaves the previous contents exactly as they were, so the
|
|
// old good copy survives until a complete new one is available to
|
|
// replace it in a single rename.
|
|
const existing = new TextEncoder().encode(
|
|
"previously downloaded, known-good contents",
|
|
);
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptNonFinalBody(
|
|
patternBytes(256, 22),
|
|
key,
|
|
);
|
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "existing.bin");
|
|
writeFileSync(outPath, existing);
|
|
|
|
await expect(download(api, file, outPath)).rejects.toBeInstanceOf(
|
|
TruncatedStreamError,
|
|
);
|
|
|
|
expect(readFileSync(outPath)).toEqual(Buffer.from(existing));
|
|
expect(readdirSync(dir)).toEqual(["existing.bin"]);
|
|
});
|
|
|
|
it("replaces an existing file when the download succeeds", async () => {
|
|
// The mirror image of the previous test: a complete download does
|
|
// overwrite whatever was at the destination, atomically, via rename.
|
|
const existing = new TextEncoder().encode("stale contents");
|
|
const plaintext = patternBytes(512, 23);
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
|
|
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "replaced.bin");
|
|
writeFileSync(outPath, existing);
|
|
|
|
const result = await download(api, file, outPath);
|
|
|
|
expect(result).toEqual({ path: outPath, bytesWritten: 512 });
|
|
expect(readFileSync(outPath)).toEqual(Buffer.from(plaintext));
|
|
// The temp file is gone once the rename has happened, so a
|
|
// successful download leaves exactly one file behind.
|
|
expect(readdirSync(dir)).toEqual(["replaced.bin"]);
|
|
});
|
|
|
|
it("stages the plaintext in a sibling temp file and renames it into place", async () => {
|
|
// The atomic write, observed directly rather than inferred from an
|
|
// empty directory. Every other failure in this file is injected
|
|
// inside `streamDecrypt`, which runs before anything is written —
|
|
// so under those tests a plain `writeFile` to the destination would
|
|
// look identical. This one watches the rename itself.
|
|
//
|
|
// Two properties are load-bearing. The staging file must exist on
|
|
// disk when the rename happens: that is what makes the destination
|
|
// appear complete or not at all, instead of filling up as bytes
|
|
// land. And it must be a sibling of the destination, because
|
|
// `rename` is only atomic within one filesystem — staging in
|
|
// `/tmp` and renaming across a mount point would silently become a
|
|
// copy, reintroducing the partial-file window this exists to close.
|
|
const plaintext = patternBytes(512, 31);
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "staged.bin");
|
|
|
|
await download(api, file, outPath);
|
|
|
|
expect(renameHook.calls).toHaveLength(1);
|
|
const staged = renameHook.calls[0]!;
|
|
expect(staged.to).toBe(outPath);
|
|
expect(staged.from).not.toBe(outPath);
|
|
expect(dirname(staged.from)).toBe(dir);
|
|
expect(staged.sourceExisted).toBe(true);
|
|
|
|
// Afterwards the temp file is gone and only the destination is
|
|
// left, holding the complete plaintext.
|
|
expect(existsSync(staged.from)).toBe(false);
|
|
expect(readFileSync(outPath)).toEqual(Buffer.from(plaintext));
|
|
expect(readdirSync(dir)).toEqual(["staged.bin"]);
|
|
});
|
|
|
|
it("removes the staged temp file when the rename itself fails", async () => {
|
|
// The cleanup path. It can only run when something fails at or
|
|
// after the write, which no amount of bad network data can
|
|
// produce: by the time anything is written the whole stream has
|
|
// already decrypted and verified. Failing the rename is what
|
|
// reaches it — a real possibility on a full disk, a read-only
|
|
// mount, or a destination that has become a directory.
|
|
//
|
|
// This is the only case in which the temp file is on disk at the
|
|
// moment of failure, so it is the only one that can show it is
|
|
// actually removed rather than merely never created. It also pins
|
|
// that the caller sees the original failure: a cleanup that threw
|
|
// over the top of it would hide why the download failed.
|
|
const existing = new TextEncoder().encode("known-good contents");
|
|
const plaintext = patternBytes(512, 32);
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "rename-fails.bin");
|
|
writeFileSync(outPath, existing);
|
|
|
|
renameHook.failWith = new Error("simulated rename failure");
|
|
|
|
await expect(download(api, file, outPath)).rejects.toThrow(
|
|
"simulated rename failure",
|
|
);
|
|
|
|
expect(renameHook.calls).toHaveLength(1);
|
|
const staged = renameHook.calls[0]!;
|
|
expect(staged.sourceExisted).toBe(true);
|
|
expect(existsSync(staged.from)).toBe(false);
|
|
|
|
// The previous contents are still there, untouched, and the
|
|
// directory holds nothing else.
|
|
expect(readFileSync(outPath)).toEqual(Buffer.from(existing));
|
|
expect(readdirSync(dir)).toEqual(["rename-fails.bin"]);
|
|
});
|
|
|
|
it("fails without creating anything when the destination directory does not exist", async () => {
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(
|
|
patternBytes(64, 33),
|
|
key,
|
|
);
|
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "missing", "never.bin");
|
|
|
|
await expect(download(api, file, outPath)).rejects.toMatchObject({
|
|
code: "ENOENT",
|
|
});
|
|
|
|
// The missing directory is not created on the caller's behalf.
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
});
|
|
|
|
// Root ignores directory permissions, so this cannot fail as root
|
|
// (the Docker test image runs as root).
|
|
it.skipIf(process.getuid?.() === 0)(
|
|
"fails without creating anything when the destination directory is not writable",
|
|
async () => {
|
|
const key =
|
|
sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(
|
|
patternBytes(64, 34),
|
|
key,
|
|
);
|
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "never.bin");
|
|
chmodSync(dir, 0o500);
|
|
try {
|
|
await expect(
|
|
download(api, file, outPath),
|
|
).rejects.toMatchObject({ code: "EACCES" });
|
|
} finally {
|
|
chmodSync(dir, 0o700);
|
|
}
|
|
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
},
|
|
);
|
|
},
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Retries
|
|
//
|
|
// What is retried here is the whole download — request, stream consumption,
|
|
// decryption — because only the first of those three happens inside
|
|
// `ApiClient`. Every assertion counts requests; none of them measures time.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe.each(entryPoints)("$name retries", ({ name, download }) => {
|
|
const freshDir = (): string => mkdtempSync(join(testDir, `${name}-retry-`));
|
|
|
|
/** A cheap single-chunk fixture: no 4 MiB encryption in the retry tests. */
|
|
const smallFixture = (
|
|
seed: number,
|
|
): {
|
|
key: Uint8Array;
|
|
header: Uint8Array;
|
|
ciphertext: Uint8Array;
|
|
plaintext: Uint8Array;
|
|
} => {
|
|
const plaintext = patternBytes(1024, seed);
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
|
return { key, header, ciphertext, plaintext };
|
|
};
|
|
|
|
const clientFor = (
|
|
fetch: typeof globalThis.fetch,
|
|
attempts: number,
|
|
): ApiClient => new ApiClient({ fetch, retry: { ...noWait, attempts } });
|
|
|
|
it("retries a connection reset that happened mid-body", async () => {
|
|
// The case `ApiClient` cannot see. Its own request succeeded: headers
|
|
// arrived, a `ReadableStream` was handed back, and only then did the
|
|
// socket die. Retrying the fetch alone would have caught nothing,
|
|
// which is why the retry wraps the whole sequence.
|
|
const { key, header, ciphertext, plaintext } = smallFixture(41);
|
|
const { fetch, requests } = scriptedCdnFetch(
|
|
{ kind: "reset", bytes: ciphertext.slice(0, 16) },
|
|
{ kind: "body", bytes: ciphertext },
|
|
);
|
|
const api = clientFor(fetch, 4);
|
|
const file = buildMockEnteFile(key, header, header);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "reset-then-ok.bin");
|
|
|
|
const result = await download(api, file, outPath);
|
|
|
|
expect(requests()).toBe(2);
|
|
expect(result.bytesWritten).toBe(plaintext.length);
|
|
expectSameBytes(readFileSync(outPath), plaintext);
|
|
});
|
|
|
|
it("stages one temp file for the attempt that succeeded, not one per attempt", async () => {
|
|
// Each streaming attempt stages into its own temp file, but a retried
|
|
// download must not leave a trail of half-written scratch files: a
|
|
// failed attempt removes its temp file, and the destination is renamed
|
|
// into place exactly once — by the attempt that produced a complete,
|
|
// authenticated plaintext. (Here the two failed attempts reset before a
|
|
// whole chunk is pulled, so they write nothing; the point stands either
|
|
// way — see the retry-restart test below, where they do write.)
|
|
const { key, header, ciphertext } = smallFixture(42);
|
|
const { fetch } = scriptedCdnFetch(
|
|
{ kind: "reset", bytes: ciphertext.slice(0, 16) },
|
|
{ kind: "reset", bytes: ciphertext.slice(0, 16) },
|
|
{ kind: "body", bytes: ciphertext },
|
|
);
|
|
const api = clientFor(fetch, 4);
|
|
const file = buildMockEnteFile(key, header, header);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "one-stage.bin");
|
|
|
|
await download(api, file, outPath);
|
|
|
|
expect(renameHook.calls).toHaveLength(1);
|
|
expect(renameHook.calls[0]!.to).toBe(outPath);
|
|
expect(readdirSync(dir)).toEqual(["one-stage.bin"]);
|
|
});
|
|
|
|
it("retries a truncated body and gives up after the configured attempts", async () => {
|
|
// Truncation is retryable — the file on the server is intact, the
|
|
// transfer was not — but it is not retryable forever. Three attempts
|
|
// configured, three requests, then the caller gets the error.
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptNonFinalBody(
|
|
patternBytes(256, 43),
|
|
key,
|
|
);
|
|
const { fetch, requests } = scriptedCdnFetch(
|
|
{ kind: "body", bytes: ciphertext },
|
|
{ kind: "body", bytes: ciphertext },
|
|
{ kind: "body", bytes: ciphertext },
|
|
{ kind: "body", bytes: ciphertext },
|
|
);
|
|
const api = clientFor(fetch, 3);
|
|
const file = buildMockEnteFile(key, header, header);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "always-truncated.bin");
|
|
|
|
await expect(download(api, file, outPath)).rejects.toBeInstanceOf(
|
|
TruncatedStreamError,
|
|
);
|
|
|
|
expect(requests()).toBe(3);
|
|
// Every attempt failed before anything was written, so the directory
|
|
// is still empty.
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
});
|
|
|
|
it("issues exactly one request when the file is gone", async () => {
|
|
// A 404 from the CDN is an answer. `runBackup` logs it and moves on;
|
|
// spending three more requests and three backoff waits on it would
|
|
// slow a large backup down for nothing.
|
|
const { key, header } = smallFixture(44);
|
|
const { fetch, requests } = scriptedCdnFetch(
|
|
{ kind: "status", status: 404 },
|
|
{ kind: "status", status: 404 },
|
|
{ kind: "status", status: 404 },
|
|
{ kind: "status", status: 404 },
|
|
);
|
|
const api = clientFor(fetch, 4);
|
|
const file = buildMockEnteFile(key, header, header);
|
|
const outPath = join(freshDir(), "gone.bin");
|
|
|
|
const err: unknown = await download(api, file, outPath).catch(
|
|
(e: unknown) => e,
|
|
);
|
|
|
|
expect(err).toBeInstanceOf(ApiError);
|
|
expect((err as ApiError).status).toBe(404);
|
|
expect(requests()).toBe(1);
|
|
});
|
|
|
|
it("retries a 503 from the CDN", async () => {
|
|
const { key, header, ciphertext, plaintext } = smallFixture(45);
|
|
const { fetch, requests } = scriptedCdnFetch(
|
|
{ kind: "status", status: 503 },
|
|
{ kind: "status", status: 503 },
|
|
{ kind: "body", bytes: ciphertext },
|
|
);
|
|
const api = clientFor(fetch, 4);
|
|
const file = buildMockEnteFile(key, header, header);
|
|
const outPath = join(freshDir(), "flaky-cdn.bin");
|
|
|
|
await download(api, file, outPath);
|
|
|
|
expect(requests()).toBe(3);
|
|
expectSameBytes(readFileSync(outPath), plaintext);
|
|
});
|
|
|
|
it("spends one attempt budget, not one per layer", async () => {
|
|
// `ApiClient.getFileStream` retries on its own for direct callers.
|
|
// The download layer opts out of that and runs its own retry over the
|
|
// whole sequence. If it did not, the two budgets would compose: three
|
|
// attempts here would become nine requests to the CDN for a single
|
|
// file, and the default four would become sixteen.
|
|
const { key, header } = smallFixture(46);
|
|
const steps: BodyStep[] = Array.from({ length: 12 }, () => ({
|
|
kind: "status" as const,
|
|
status: 503,
|
|
}));
|
|
const { fetch, requests } = scriptedCdnFetch(...steps);
|
|
const api = clientFor(fetch, 3);
|
|
const file = buildMockEnteFile(key, header, header);
|
|
const outPath = join(freshDir(), "budget.bin");
|
|
|
|
await expect(download(api, file, outPath)).rejects.toBeInstanceOf(
|
|
ApiError,
|
|
);
|
|
|
|
expect(requests()).toBe(3);
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Streaming decrypt to disk
|
|
//
|
|
// The plaintext is never held whole in memory: each secretstream chunk is
|
|
// written to the temp file as it is decrypted, so peak memory is bounded by the
|
|
// chunk size rather than the file size. These tests watch the writes directly
|
|
// (see `writeHook`) rather than infer memory behaviour from the final file.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe.each(entryPoints)("$name streams to disk", ({ name, download }) => {
|
|
const freshDir = (): string =>
|
|
mkdtempSync(join(testDir, `${name}-stream-`));
|
|
|
|
/** Writes recorded against staged temp files (not the `writeFile` path). */
|
|
const tempWrites = (): { path: string; length: number }[] =>
|
|
writeHook.writes.filter((w) => w.path.endsWith(".tmp"));
|
|
|
|
it("writes one chunk at a time, none larger than STREAM_CHUNK_SIZE", async () => {
|
|
// The multi-chunk fixture decrypts to one full 4 MiB chunk plus a small
|
|
// final chunk. A streaming writer therefore issues exactly two writes,
|
|
// of STREAM_CHUNK_SIZE and then the final chunk's length — never a
|
|
// single write carrying the whole 4 MiB + 1 KiB file. That per-chunk
|
|
// shape is what "memory bounded by chunk size" means in practice: the
|
|
// plaintext is handed to the filesystem and dropped, chunk by chunk.
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.body,
|
|
);
|
|
const outPath = join(freshDir(), "streamed.bin");
|
|
|
|
const result = await download(api, file, outPath);
|
|
|
|
const writes = tempWrites();
|
|
expect(writes.map((w) => w.length)).toEqual([
|
|
STREAM_CHUNK_SIZE,
|
|
multiChunk.plaintext.length - STREAM_CHUNK_SIZE,
|
|
]);
|
|
// No single write ever carried the whole file, and every write fits in
|
|
// one chunk's worth of memory.
|
|
for (const w of writes) {
|
|
expect(w.length).toBeLessThanOrEqual(STREAM_CHUNK_SIZE);
|
|
}
|
|
expect(result.bytesWritten).toBe(multiChunk.plaintext.length);
|
|
expectSameBytes(readFileSync(outPath), multiChunk.plaintext);
|
|
});
|
|
|
|
it("restarts from byte zero on a retry, replacing the temp file cleanly", async () => {
|
|
// The secretstream pull state is not resumable, so a retry cannot
|
|
// continue a half-written file — it must start over. The first attempt
|
|
// here delivers a complete leading chunk and then stops before the
|
|
// TAG_FINAL chunk: 4 MiB of plaintext lands in a temp file, then the
|
|
// download is rejected as truncated and that temp file is discarded.
|
|
// The retry streams the whole body into a *fresh* temp file, so the
|
|
// destination ends up with exactly the plaintext once — never the
|
|
// leading chunk twice, and never a stale temp file left behind.
|
|
const truncatedBody = multiChunk.body.slice(
|
|
0,
|
|
multiChunk.finalChunkOffset,
|
|
);
|
|
const { fetch, requests } = scriptedCdnFetch(
|
|
{ kind: "body", bytes: truncatedBody },
|
|
{ kind: "body", bytes: multiChunk.body },
|
|
);
|
|
const api = new ApiClient({
|
|
fetch,
|
|
retry: { ...noWait, attempts: 4 },
|
|
});
|
|
const file = buildMockEnteFile(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.header,
|
|
);
|
|
const dir = freshDir();
|
|
const outPath = join(dir, "retry-restart.bin");
|
|
|
|
const result = await download(api, file, outPath);
|
|
|
|
expect(requests()).toBe(2);
|
|
// Both attempts streamed to disk, each into its own temp file: the
|
|
// truncated first attempt wrote before it failed, proving the retry did
|
|
// not resume a partial file but replaced it.
|
|
const distinctTemps = new Set(tempWrites().map((w) => w.path));
|
|
expect(distinctTemps.size).toBe(2);
|
|
// The destination holds the complete plaintext exactly once, and no
|
|
// temp file survives.
|
|
expect(result.bytesWritten).toBe(multiChunk.plaintext.length);
|
|
expectSameBytes(readFileSync(outPath), multiChunk.plaintext);
|
|
expect(renameHook.calls).toHaveLength(1);
|
|
expect(readdirSync(dir)).toEqual(["retry-restart.bin"]);
|
|
});
|
|
});
|
|
|
|
describe("download retries: corruption is not retried", () => {
|
|
it("gives up immediately on a chunk that failed to authenticate", async () => {
|
|
// A whole chunk that failed to authenticate while the stream
|
|
// continued past it is corruption or a wrong key. Neither is fixed by
|
|
// asking again, and a backup run that retried every such file would
|
|
// multiply the cost of a genuinely broken file by the attempt count.
|
|
//
|
|
// This is also the boundary of the single-chunk ambiguity documented
|
|
// at the classifier: the split is only achievable because this body
|
|
// has more than one chunk.
|
|
const corrupted = Uint8Array.from(multiChunk.body);
|
|
corrupted[10] ^= 0xff;
|
|
const { fetch, requests } = scriptedCdnFetch(
|
|
{ kind: "body", bytes: corrupted },
|
|
{ kind: "body", bytes: corrupted },
|
|
{ kind: "body", bytes: corrupted },
|
|
{ kind: "body", bytes: corrupted },
|
|
);
|
|
const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } });
|
|
const file = buildMockEnteFile(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.header,
|
|
);
|
|
const outPath = join(mkdtempSync(join(testDir, "corrupt-")), "c.bin");
|
|
|
|
await expect(downloadFile(api, file, outPath)).rejects.toThrow(
|
|
/authentication failed/i,
|
|
);
|
|
|
|
expect(requests()).toBe(1);
|
|
});
|
|
|
|
it("cancels the response body when decryption fails", async () => {
|
|
// A backup run carries on past a failed file, so a body left open on
|
|
// failure would hold its connection until garbage collection, once
|
|
// per failed file. This body delivers a corrupt chunk and then stays
|
|
// open, so only a cancel from the downloader can close it.
|
|
const corrupted = Uint8Array.from(multiChunk.body);
|
|
corrupted[10] ^= 0xff;
|
|
let cancelled = false;
|
|
const fetch = (async () =>
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(corrupted);
|
|
},
|
|
cancel() {
|
|
cancelled = true;
|
|
},
|
|
}),
|
|
{ status: 200 },
|
|
)) as typeof globalThis.fetch;
|
|
const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 1 } });
|
|
const file = buildMockEnteFile(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.header,
|
|
);
|
|
const outPath = join(mkdtempSync(join(testDir, "cancel-")), "c.bin");
|
|
|
|
await expect(downloadFile(api, file, outPath)).rejects.toThrow(
|
|
/authentication failed/i,
|
|
);
|
|
|
|
expect(cancelled).toBe(true);
|
|
});
|
|
|
|
it("cancels the response body when the temp file cannot be opened", async () => {
|
|
// The download fails before a byte of the body is read, so the body
|
|
// is still open and only a cancel from the downloader can close it.
|
|
let cancelled = false;
|
|
const fetch = (async () =>
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(multiChunk.body);
|
|
},
|
|
cancel() {
|
|
cancelled = true;
|
|
},
|
|
}),
|
|
{ status: 200 },
|
|
)) as typeof globalThis.fetch;
|
|
const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 1 } });
|
|
const file = buildMockEnteFile(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.header,
|
|
);
|
|
const outPath = join(
|
|
mkdtempSync(join(testDir, "cancel-")),
|
|
"missing",
|
|
"c.bin",
|
|
);
|
|
|
|
await expect(downloadFile(api, file, outPath)).rejects.toMatchObject({
|
|
code: "ENOENT",
|
|
});
|
|
|
|
expect(cancelled).toBe(true);
|
|
});
|
|
|
|
it("cancels the response body when the header is malformed", async () => {
|
|
// The header is rejected before the body is read, so the body is
|
|
// still open and only a cancel from the downloader can close it.
|
|
let cancelled = false;
|
|
const fetch = (async () =>
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(multiChunk.body);
|
|
},
|
|
cancel() {
|
|
cancelled = true;
|
|
},
|
|
}),
|
|
{ status: 200 },
|
|
)) as typeof globalThis.fetch;
|
|
const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 1 } });
|
|
const shortHeader = multiChunk.header.subarray(0, 5);
|
|
const file = buildMockEnteFile(multiChunkKey, shortHeader, shortHeader);
|
|
const outPath = join(mkdtempSync(join(testDir, "cancel-")), "c.bin");
|
|
|
|
await expect(downloadFile(api, file, outPath)).rejects.toThrow();
|
|
|
|
expect(cancelled).toBe(true);
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Fragmented network reads
|
|
//
|
|
// A CDN does not hand the body over one secretstream chunk at a time; it
|
|
// arrives in whatever pieces the socket produces, many of them far smaller than
|
|
// a chunk and most straddling a chunk boundary. `streamDecrypt` reassembles
|
|
// those pieces before decrypting, copying each received byte once rather than
|
|
// recopying the whole accumulator on every read. This is the path the other
|
|
// fixtures never take — their mock fetch delivers each body as a single
|
|
// `Response` value, i.e. one read — so it is exercised explicitly here.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* A fetch that serves `body` through a `ReadableStream` sliced into many
|
|
* fixed-size pieces, imitating a socket that trickles bytes in. `pieceSize` is
|
|
* chosen not to divide the chunk framing evenly, so pieces straddle the
|
|
* `ENC_CHUNK_SIZE` boundary the downloader splits on — the case a single-value
|
|
* body can never produce. `emitted` reports how many pieces were yielded, so a
|
|
* test can assert the body really was fragmented and not delivered whole.
|
|
*/
|
|
const mockFetchForFragmentedBody = (
|
|
body: Uint8Array,
|
|
pieceSize: number,
|
|
): { fetch: typeof globalThis.fetch; emitted: () => number } => {
|
|
let pieces = 0;
|
|
const fake = async (): Promise<Response> =>
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
for (let off = 0; off < body.length; off += pieceSize) {
|
|
controller.enqueue(body.subarray(off, off + pieceSize));
|
|
pieces++;
|
|
}
|
|
controller.close();
|
|
},
|
|
}),
|
|
{ status: 200 },
|
|
);
|
|
return { fetch: fake as typeof globalThis.fetch, emitted: () => pieces };
|
|
};
|
|
|
|
describe("streamDecrypt fragmented reads", () => {
|
|
it("decrypts a multi-chunk body delivered in many small pieces", async () => {
|
|
// The multi-chunk fixture (one full 4 MiB chunk plus a small final
|
|
// chunk) delivered in 1000-byte pieces: several thousand reads, with
|
|
// the piece that spans the 4 MiB + 17 byte chunk boundary split across
|
|
// two chunks by the reassembler. The plaintext must come out
|
|
// byte-identical to the single-read case, and the chunk framing must be
|
|
// untouched: exactly two writes, `STREAM_CHUNK_SIZE` then the final
|
|
// chunk, the same as when the body arrives whole. If the boundary
|
|
// handling were off by a byte under fragmentation, either the pull
|
|
// would fail to authenticate or the write sizes would shift.
|
|
const { fetch, emitted } = mockFetchForFragmentedBody(
|
|
multiChunk.body,
|
|
1000,
|
|
);
|
|
const api = new ApiClient({ fetch });
|
|
const file = buildMockEnteFile(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.header,
|
|
);
|
|
const dir = mkdtempSync(join(testDir, "fragmented-"));
|
|
const outPath = join(dir, "fragmented.bin");
|
|
|
|
const result = await downloadFile(api, file, outPath);
|
|
|
|
// The body really was trickled in, not handed over whole.
|
|
expect(emitted()).toBeGreaterThan(1000);
|
|
|
|
const writes = writeHook.writes.filter((w) => w.path.endsWith(".tmp"));
|
|
expect(writes.map((w) => w.length)).toEqual([
|
|
STREAM_CHUNK_SIZE,
|
|
multiChunk.plaintext.length - STREAM_CHUNK_SIZE,
|
|
]);
|
|
expect(result.bytesWritten).toBe(multiChunk.plaintext.length);
|
|
expectSameBytes(readFileSync(outPath), multiChunk.plaintext);
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Durable atomic writes
|
|
//
|
|
// `writeAtomic` is exported so the metadata store can reuse the same
|
|
// power-cut-safe write. Its durability is the point: the bytes and the new
|
|
// directory entry must both be on stable storage before it returns, so a crash
|
|
// immediately afterwards cannot resurrect an empty renamed file (#22 area 1).
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe("writeAtomic", () => {
|
|
it("fsyncs the temp file before the rename and the directory after", async () => {
|
|
const dir = mkdtempSync(join(testDir, "atomic-"));
|
|
const dest = join(dir, "durable.bin");
|
|
const bytes = patternBytes(2048, 71);
|
|
|
|
await writeAtomic(dest, bytes);
|
|
|
|
expect(readFileSync(dest)).toEqual(Buffer.from(bytes));
|
|
// The order is the durability contract: fsync the staged temp file so
|
|
// its contents are on disk, rename it into place, then fsync the
|
|
// directory so that new entry is on disk too. Do the directory fsync
|
|
// before the rename, or skip it, and a crash can lose the rename.
|
|
expect(durabilityHook.events).toHaveLength(3);
|
|
expect(durabilityHook.events[0]).toMatch(/^sync:w:.*\.tmp$/);
|
|
expect(durabilityHook.events[1]).toBe(`rename:${dest}`);
|
|
expect(durabilityHook.events[2]).toBe(`sync:r:${dir}`);
|
|
});
|
|
|
|
it("leaves no temp file behind when the write cannot be renamed", async () => {
|
|
const dir = mkdtempSync(join(testDir, "atomic-fail-"));
|
|
const dest = join(dir, "unrenamable.bin");
|
|
renameHook.failWith = new Error("simulated rename failure");
|
|
|
|
await expect(writeAtomic(dest, patternBytes(64, 72))).rejects.toThrow(
|
|
"simulated rename failure",
|
|
);
|
|
|
|
// The staged temp file was fsynced, then the rename failed; the cleanup
|
|
// path must remove it so a repeatedly failing write cannot fill the disk.
|
|
expect(existsSync(dest)).toBe(false);
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Per-chunk progress
|
|
//
|
|
// Callers streaming a large file want bytes-written as it lands, not only the
|
|
// final total. The hook fires as decrypted plaintext accumulates; its values
|
|
// are non-decreasing and its last value is exactly `bytesWritten`.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe.each(entryPoints)("$name progress", ({ name, download }) => {
|
|
it("reports monotonic progress ending at bytesWritten", async () => {
|
|
// The multi-chunk fixture pulls one full 4 MiB chunk and then a small
|
|
// final chunk, so the callback fires more than once and monotonicity is
|
|
// actually observable rather than trivially true for a single fire.
|
|
const { api, file } = fixtureFor(
|
|
multiChunkKey,
|
|
multiChunk.header,
|
|
multiChunk.body,
|
|
);
|
|
const outPath = join(
|
|
mkdtempSync(join(testDir, `${name}-progress-`)),
|
|
"p.bin",
|
|
);
|
|
const seen: number[] = [];
|
|
|
|
const result = await download(api, file, outPath, (bytesDone) => {
|
|
seen.push(bytesDone);
|
|
});
|
|
|
|
expect(seen.length).toBeGreaterThan(1);
|
|
for (let i = 1; i < seen.length; i++) {
|
|
expect(seen[i]!).toBeGreaterThan(seen[i - 1]!);
|
|
}
|
|
expect(seen[seen.length - 1]).toBe(result.bytesWritten);
|
|
expect(result.bytesWritten).toBe(multiChunk.plaintext.length);
|
|
});
|
|
|
|
it("downloads normally when no progress callback is given", async () => {
|
|
// The callback is optional and its absence must be side-effect-free:
|
|
// the download succeeds exactly as it does elsewhere in this file.
|
|
const plaintext = patternBytes(300, 73);
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
|
const outPath = join(
|
|
mkdtempSync(join(testDir, `${name}-noprog-`)),
|
|
"n.bin",
|
|
);
|
|
|
|
const result = await download(api, file, outPath);
|
|
|
|
expect(result.bytesWritten).toBe(plaintext.length);
|
|
expectSameBytes(readFileSync(outPath), plaintext);
|
|
});
|
|
});
|
|
|
|
describe("downloadFile content hash", () => {
|
|
// Node's own BLAKE2b-512 is the reference, so these tests do not depend
|
|
// on the code under test to compute what they expect.
|
|
const blake2b = (bytes: Uint8Array): string =>
|
|
createHash("blake2b512").update(bytes).digest("base64");
|
|
|
|
// Serve `plaintext` encrypted as file 999 with the given metadata. Four
|
|
// responses are scripted so a retried mismatch would show in `requests`.
|
|
const setup = (plaintext: Uint8Array, metadata: Partial<FileMetadata>) => {
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
|
const file = buildMockEnteFile(key, header, header);
|
|
file.metadata = { ...file.metadata, ...metadata };
|
|
const body = { kind: "body", bytes: ciphertext } as const;
|
|
const { fetch, requests } = scriptedCdnFetch(body, body, body, body);
|
|
const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } });
|
|
const dir = mkdtempSync(join(testDir, "hash-"));
|
|
const outPath = join(dir, "f.bin");
|
|
return {
|
|
run: () => downloadFile(api, file, outPath),
|
|
dir,
|
|
outPath,
|
|
requests,
|
|
};
|
|
};
|
|
|
|
const livePhotoZip = zipSync({
|
|
"image.heic": patternBytes(500, 81),
|
|
"video.mov": patternBytes(900, 82),
|
|
});
|
|
const livePhotoHash = `${blake2b(patternBytes(500, 81))}:${blake2b(patternBytes(900, 82))}`;
|
|
|
|
it("stores a file whose hash matches", async () => {
|
|
const plaintext = patternBytes(700, 80);
|
|
const t = setup(plaintext, { hash: blake2b(plaintext) });
|
|
|
|
await t.run();
|
|
|
|
expectSameBytes(readFileSync(t.outPath), plaintext);
|
|
});
|
|
|
|
it("rejects a mismatch, stores nothing, names the file and does not retry", async () => {
|
|
const t = setup(patternBytes(700, 80), {
|
|
hash: blake2b(patternBytes(700, 79)),
|
|
});
|
|
|
|
await expect(t.run()).rejects.toThrow(
|
|
/file 999: content hash .* does not match/,
|
|
);
|
|
|
|
expect(readdirSync(t.dir)).toEqual([]);
|
|
expect(t.requests()).toBe(1);
|
|
});
|
|
|
|
it("stores a file with no recorded hash unchecked", async () => {
|
|
const plaintext = patternBytes(700, 80);
|
|
const t = setup(plaintext, { hash: undefined });
|
|
|
|
await t.run();
|
|
|
|
expectSameBytes(readFileSync(t.outPath), plaintext);
|
|
});
|
|
|
|
it("stores a live photo whose image and video hashes match", async () => {
|
|
const t = setup(livePhotoZip, {
|
|
fileType: "livePhoto",
|
|
hash: livePhotoHash,
|
|
});
|
|
|
|
await t.run();
|
|
|
|
expectSameBytes(readFileSync(t.outPath), livePhotoZip);
|
|
});
|
|
|
|
it("hashes a large live photo entry as it decompresses, never whole", async () => {
|
|
// 64 MiB of zeros deflates to a few kilobytes, the shape of a ZIP
|
|
// that would exhaust memory if expanded whole.
|
|
const image = new Uint8Array(64 * 1024 * 1024);
|
|
const video = patternBytes(900, 83);
|
|
const zip = zipSync({ "image.heic": image, "video.mov": video });
|
|
const t = setup(zip, {
|
|
fileType: "livePhoto",
|
|
hash: `${blake2b(image)}:${blake2b(video)}`,
|
|
});
|
|
|
|
await t.run();
|
|
|
|
expectSameBytes(readFileSync(t.outPath), zip);
|
|
const hashed = hashHook.lengths.reduce((a, b) => a + b, 0);
|
|
expect(hashed).toBe(image.length + video.length);
|
|
expect(Math.max(...hashHook.lengths)).toBeLessThanOrEqual(
|
|
2 * STREAM_CHUNK_SIZE,
|
|
);
|
|
});
|
|
|
|
it("rejects a live photo whose hash does not match", async () => {
|
|
// The whole ZIP's hash is not the recorded one: each part is hashed.
|
|
const t = setup(livePhotoZip, {
|
|
fileType: "livePhoto",
|
|
hash: blake2b(livePhotoZip),
|
|
});
|
|
|
|
await expect(t.run()).rejects.toThrow(
|
|
/file 999: content hash .* does not match/,
|
|
);
|
|
|
|
expect(readdirSync(t.dir)).toEqual([]);
|
|
});
|
|
});
|