Stream decrypted downloads to disk with bounded memory (closes #40)

Originals no longer buffer the whole decrypted file in RAM. `streamDecrypt`
writes each secretstream chunk to the staged temp file as it is pulled and
returns the byte count, so peak memory is one chunk, not the file size. The
temp-then-rename fsync discipline of the exported `writeAtomic` is factored
into a shared helper that both the whole-buffer path and the streaming path
use.

The rename still happens only after the stream authenticates on `TAG_FINAL`;
a truncated or corrupt stream throws and removes the temp file, leaving the
destination untouched as before. Because the plaintext is no longer buffered,
the atomic write moved inside the retry: each attempt streams from byte zero
into its own temp file and only a complete attempt renames.

Closes #21.

Model: opus-4-8

test/download/download.test.ts

@@ -118,6 +118,21 @@ const durabilityHook = vi.hoisted(() => ({
     events: [] as string[],
 }));
 
+/**
+ * `FileHandle.write` is wrapped so the tests can watch the streaming decrypt
+ * path put plaintext on disk one chunk at a time. This is the direct evidence
+ * that memory is bounded by the chunk size and not the file size: a buffered
+ * downloader would hand the whole file to a single write, whereas the streaming
+ * one issues one write per secretstream chunk, none larger than
+ * `STREAM_CHUNK_SIZE`. Each write records the temp path it targeted and its
+ * length. `writeFile` (which the whole-buffer `writeAtomic` uses) is a distinct
+ * native call and does not go through this method, so only the streaming path
+ * is observed here.
+ */
+const writeHook = vi.hoisted(() => ({
+    writes: [] as { path: string; length: number }[],
+}));
+
 vi.mock("node:fs/promises", async (importOriginal) => {
     const actual = await importOriginal<typeof import("node:fs/promises")>();
     const { existsSync: sourceExists } = await import("node:fs");
@@ -138,6 +153,22 @@ vi.mock("node:fs/promises", async (importOriginal) => {
                 durabilityHook.events.push(`sync:${String(flags)}:${path}`);
                 await realSync();
             };
+            const realWrite = handle.write.bind(handle);
+            handle.write = (async (
+                data: unknown,
+                ...rest2: unknown[]
+            ): Promise<unknown> => {
+                if (data instanceof Uint8Array) {
+                    writeHook.writes.push({
+                        path: String(path),
+                        length: data.length,
+                    });
+                }
+                return (realWrite as (...a: unknown[]) => Promise<unknown>)(
+                    data,
+                    ...rest2,
+                );
+            }) as typeof handle.write;
             return handle;
         },
         rename: async (from: string, to: string): Promise<void> => {
@@ -159,6 +190,7 @@ beforeEach(() => {
     renameHook.calls.length = 0;
     renameHook.failWith = null;
     durabilityHook.events.length = 0;
+    writeHook.writes.length = 0;
 });
 
 let testDir: string;
@@ -947,10 +979,13 @@ describe.each(entryPoints)("$name retries", ({ name, download }) => {
     });
 
     it("stages one temp file for the attempt that succeeded, not one per attempt", async () => {
-        // The atomic write stays outside the retry loop. A retried download
-        // must not leave a trail of half-written scratch files, and the
-        // destination must be touched exactly once — by the attempt that
-        // produced a complete, authenticated plaintext.
+        // Each streaming attempt stages into its own temp file, but a retried
+        // download must not leave a trail of half-written scratch files: a
+        // failed attempt removes its temp file, and the destination is renamed
+        // into place exactly once — by the attempt that produced a complete,
+        // authenticated plaintext. (Here the two failed attempts reset before a
+        // whole chunk is pulled, so they write nothing; the point stands either
+        // way — see the retry-restart test below, where they do write.)
         const { key, header, ciphertext } = smallFixture(42);
         const { fetch } = scriptedCdnFetch(
             { kind: "reset", bytes: ciphertext.slice(0, 16) },
@@ -1064,6 +1099,99 @@ describe.each(entryPoints)("$name retries", ({ name, download }) => {
     });
 });
 
+// ---------------------------------------------------------------------------
+// Streaming decrypt to disk
+//
+// The plaintext is never held whole in memory: each secretstream chunk is
+// written to the temp file as it is decrypted, so peak memory is bounded by the
+// chunk size rather than the file size. These tests watch the writes directly
+// (see `writeHook`) rather than infer memory behaviour from the final file.
+// ---------------------------------------------------------------------------
+
+describe.each(entryPoints)("$name streams to disk", ({ name, download }) => {
+    const freshDir = (): string =>
+        mkdtempSync(join(testDir, `${name}-stream-`));
+
+    /** Writes recorded against staged temp files (not the `writeFile` path). */
+    const tempWrites = (): { path: string; length: number }[] =>
+        writeHook.writes.filter((w) => w.path.endsWith(".tmp"));
+
+    it("writes one chunk at a time, none larger than STREAM_CHUNK_SIZE", async () => {
+        // The multi-chunk fixture decrypts to one full 4 MiB chunk plus a small
+        // final chunk. A streaming writer therefore issues exactly two writes,
+        // of STREAM_CHUNK_SIZE and then the final chunk's length — never a
+        // single write carrying the whole 4 MiB + 1 KiB file. That per-chunk
+        // shape is what "memory bounded by chunk size" means in practice: the
+        // plaintext is handed to the filesystem and dropped, chunk by chunk.
+        const { api, file } = fixtureFor(
+            multiChunkKey,
+            multiChunk.header,
+            multiChunk.body,
+        );
+        const outPath = join(freshDir(), "streamed.bin");
+
+        const result = await download(api, file, outPath);
+
+        const writes = tempWrites();
+        expect(writes.map((w) => w.length)).toEqual([
+            STREAM_CHUNK_SIZE,
+            multiChunk.plaintext.length - STREAM_CHUNK_SIZE,
+        ]);
+        // No single write ever carried the whole file, and every write fits in
+        // one chunk's worth of memory.
+        for (const w of writes) {
+            expect(w.length).toBeLessThanOrEqual(STREAM_CHUNK_SIZE);
+        }
+        expect(result.bytesWritten).toBe(multiChunk.plaintext.length);
+        expectSameBytes(readFileSync(outPath), multiChunk.plaintext);
+    });
+
+    it("restarts from byte zero on a retry, replacing the temp file cleanly", async () => {
+        // The secretstream pull state is not resumable, so a retry cannot
+        // continue a half-written file — it must start over. The first attempt
+        // here delivers a complete leading chunk and then stops before the
+        // TAG_FINAL chunk: 4 MiB of plaintext lands in a temp file, then the
+        // download is rejected as truncated and that temp file is discarded.
+        // The retry streams the whole body into a *fresh* temp file, so the
+        // destination ends up with exactly the plaintext once — never the
+        // leading chunk twice, and never a stale temp file left behind.
+        const truncatedBody = multiChunk.body.slice(
+            0,
+            multiChunk.finalChunkOffset,
+        );
+        const { fetch, requests } = scriptedCdnFetch(
+            { kind: "body", bytes: truncatedBody },
+            { kind: "body", bytes: multiChunk.body },
+        );
+        const api = new ApiClient({
+            fetch,
+            retry: { ...noWait, attempts: 4 },
+        });
+        const file = buildMockEnteFile(
+            multiChunkKey,
+            multiChunk.header,
+            multiChunk.header,
+        );
+        const dir = freshDir();
+        const outPath = join(dir, "retry-restart.bin");
+
+        const result = await download(api, file, outPath);
+
+        expect(requests()).toBe(2);
+        // Both attempts streamed to disk, each into its own temp file: the
+        // truncated first attempt wrote before it failed, proving the retry did
+        // not resume a partial file but replaced it.
+        const distinctTemps = new Set(tempWrites().map((w) => w.path));
+        expect(distinctTemps.size).toBe(2);
+        // The destination holds the complete plaintext exactly once, and no
+        // temp file survives.
+        expect(result.bytesWritten).toBe(multiChunk.plaintext.length);
+        expectSameBytes(readFileSync(outPath), multiChunk.plaintext);
+        expect(renameHook.calls).toHaveLength(1);
+        expect(readdirSync(dir)).toEqual(["retry-restart.bin"]);
+    });
+});
+
 describe("download retries: corruption is not retried", () => {
     it("gives up immediately on a chunk that failed to authenticate", async () => {
         // A whole chunk that failed to authenticate while the stream