Stream decrypted downloads to disk with bounded memory (closes #40)

Originals no longer buffer the whole decrypted file in RAM. `streamDecrypt`
writes each secretstream chunk to the staged temp file as it is pulled and
returns the byte count, so peak memory is one chunk, not the file size. The
temp-then-rename fsync discipline of the exported `writeAtomic` is factored
into a shared helper that both the whole-buffer path and the streaming path
use.

The rename still happens only after the stream authenticates on `TAG_FINAL`;
a truncated or corrupt stream throws and removes the temp file, leaving the
destination untouched as before. Because the plaintext is no longer buffered,
the atomic write moved inside the retry: each attempt streams from byte zero
into its own temp file and only a complete attempt renames.

Model: opus-4-8

src/download/index.ts

@@ -1,5 +1,6 @@
 import { randomUUID } from "node:crypto";
 import { open, rename, rm } from "node:fs/promises";
+import type { FileHandle } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import {
     fromBase64,
@@ -28,20 +29,44 @@ export type ProgressCallback = (bytesDone: number) => void;
 
 const ENC_CHUNK_SIZE = STREAM_CHUNK_SIZE + STREAM_CHUNK_OVERHEAD;
 
+// Decrypt a secretstream body, handing each plaintext chunk to `sink` as it is
+// produced rather than accumulating the whole file. Peak memory is one
+// ciphertext chunk of network buffer plus one plaintext chunk — bounded by
+// `STREAM_CHUNK_SIZE` regardless of the file's size — so a multi-gigabyte video
+// no longer needs its size again in RAM. Returns the total plaintext length.
+//
+// The truncation contract is exactly the buffered version's, only the sink is
+// new: a body cut short still decrypts and authenticates up to its last whole
+// chunk, so the absence of TAG_FINAL is the sole evidence it was cut short, and
+// this throws rather than let a caller keep a short file. The sink has already
+// seen those chunks by then; the caller (`decryptToTemp`) stages them in a temp
+// file that is renamed into place only on a clean return, so a throw leaves
+// nothing on disk.
 const streamDecrypt = async (
     stream: ReadableStream<Uint8Array>,
     header: Uint8Array,
     key: Uint8Array,
+    sink: (plaintext: Uint8Array) => Promise<void>,
     onProgress?: ProgressCallback,
-): Promise<Uint8Array> => {
+): Promise<number> => {
     const state = initStreamPull(header, key);
     const reader = stream.getReader();
     let buffer = new Uint8Array(0);
-    const plainChunks: Uint8Array[] = [];
     let totalPlain = 0;
     let chunksPulled = 0;
     let lastTag = -1;
 
+    const consume = async (
+        plaintext: Uint8Array,
+        tag: number,
+    ): Promise<void> => {
+        await sink(plaintext);
+        totalPlain += plaintext.length;
+        chunksPulled++;
+        lastTag = tag;
+        onProgress?.(totalPlain);
+    };
+
     for (;;) {
         const { done, value } = await reader.read();
         if (value) {
@@ -54,12 +79,10 @@ const streamDecrypt = async (
         while (buffer.length >= ENC_CHUNK_SIZE) {
             const encChunk = buffer.slice(0, ENC_CHUNK_SIZE);
             buffer = buffer.slice(ENC_CHUNK_SIZE);
+            // A whole chunk that fails to authenticate while the stream carries
+            // on is corruption, not truncation; that error propagates unchanged.
             const { plaintext, tag } = pullStreamChunk(state, encChunk);
-            plainChunks.push(plaintext);
-            totalPlain += plaintext.length;
-            chunksPulled++;
-            lastTag = tag;
-            onProgress?.(totalPlain);
+            await consume(plaintext, tag);
         }
 
         if (done) {
@@ -71,7 +94,9 @@ const streamDecrypt = async (
                 // ordinary shape of a dropped connection. Poly1305 cannot
                 // tell a partial chunk from a corrupt one, so this is
                 // reported as the truncation it almost always is, with the
-                // authentication failure kept as the error's cause.
+                // authentication failure kept as the error's cause. Only the
+                // pull is guarded: a sink failure on a chunk that did
+                // authenticate is a disk error, not a truncation.
                 let pulled;
                 try {
                     pulled = pullStreamChunk(state, buffer);
@@ -81,11 +106,7 @@ const streamDecrypt = async (
                         { cause: err },
                     );
                 }
-                plainChunks.push(pulled.plaintext);
-                totalPlain += pulled.plaintext.length;
-                chunksPulled++;
-                lastTag = pulled.tag;
-                onProgress?.(totalPlain);
+                await consume(pulled.plaintext, pulled.tag);
             }
             break;
         }
@@ -94,8 +115,6 @@ const streamDecrypt = async (
     // Only the last chunk of a secretstream carries TAG_FINAL. Everything a
     // dropped connection did deliver still decrypts and authenticates, so the
     // absence of TAG_FINAL is the only evidence that the body was cut short.
-    // Returning a short plaintext here would put a corrupt file on disk that
-    // later backup runs would treat as complete.
     if (chunksPulled === 0) {
         throw new TruncatedStreamError(
             "download: stream truncated: response body contained no secretstream chunks",
@@ -107,21 +126,16 @@ const streamDecrypt = async (
             `download: stream truncated: last chunk tag ${lastTag}, expected TAG_FINAL (${tagFinal})`,
         );
     }
-
-    const result = new Uint8Array(totalPlain);
-    let offset = 0;
-    for (const chunk of plainChunks) {
-        result.set(chunk, offset);
-        offset += chunk.length;
-    }
-    return result;
+    return totalPlain;
 };
 
-// Write `plaintext` to `destination` atomically and durably: stage it in a
-// temporary sibling file (same directory, so the rename cannot cross a
-// filesystem boundary) and rename it into place. Callers therefore never
-// observe a partially written destination, and a pre-existing file at that
-// path is replaced only once the new contents are complete on disk.
+// Stage a write to `destination` atomically and durably, then rename it into
+// place. `fill` writes the contents into the open temp file handle — either the
+// whole buffer at once (`writeAtomic`) or chunk by chunk as they decrypt
+// (`decryptToTemp`). The temp file is a sibling of the destination (same
+// directory, so the rename cannot cross a filesystem boundary), so callers
+// never observe a partially written destination, and a pre-existing file is
+// replaced only once the new contents are complete on disk.
 //
 // Durability against a power cut needs two fsyncs. Without them the write can
 // return while the data or the rename is still only in the kernel's page
@@ -131,10 +145,12 @@ const streamDecrypt = async (
 // is fsynced after it, so both the bytes and the new directory entry are on
 // stable storage before this returns.
 //
-// Exported so the metadata store can reuse the same durable write.
-export const writeAtomic = async (
+// On any failure — including a `fill` that throws because the stream was
+// truncated — the temp file is removed, so the destination is untouched and no
+// scratch file is left to fill the disk on repeated failures.
+const stageAtomic = async (
     destination: string,
-    plaintext: Uint8Array,
+    fill: (handle: FileHandle) => Promise<void>,
 ): Promise<void> => {
     const dir = dirname(destination);
     // The random suffix keeps concurrent downloads of the same destination
@@ -143,7 +159,7 @@ export const writeAtomic = async (
     try {
         const handle = await open(tmpPath, "w");
         try {
-            await handle.writeFile(plaintext);
+            await fill(handle);
             await handle.sync();
         } finally {
             await handle.close();
@@ -166,7 +182,44 @@ export const writeAtomic = async (
     }
 };
 
-// Fetch a stream and decrypt it, retrying the whole sequence.
+// Write `plaintext` to `destination` atomically and durably. Exported so the
+// metadata store can reuse the same durable write for small whole-buffer
+// payloads; originals go through `decryptToTemp` instead so they never buffer.
+export const writeAtomic = async (
+    destination: string,
+    plaintext: Uint8Array,
+): Promise<void> =>
+    stageAtomic(destination, (handle) => handle.writeFile(plaintext));
+
+// Decrypt `stream` straight to `destination`, one plaintext chunk at a time,
+// under the atomic writer's temp-then-rename discipline. Memory stays bounded
+// by the chunk size: each decrypted chunk is written to the temp file and
+// dropped. The rename happens only after the stream authenticates as terminated
+// on TAG_FINAL; a truncated stream throws and leaves the destination untouched.
+// Returns the plaintext length written.
+const decryptToTemp = async (
+    destination: string,
+    stream: ReadableStream<Uint8Array>,
+    header: Uint8Array,
+    key: Uint8Array,
+    onProgress?: ProgressCallback,
+): Promise<number> => {
+    let bytesWritten = 0;
+    await stageAtomic(destination, async (handle) => {
+        bytesWritten = await streamDecrypt(
+            stream,
+            header,
+            key,
+            async (plaintext) => {
+                await handle.write(plaintext);
+            },
+            onProgress,
+        );
+    });
+    return bytesWritten;
+};
+
+// Fetch a stream and decrypt it to `destination`, retrying the whole sequence.
 //
 // The request is only the first third of a download. `getXStream` returns as
 // soon as headers arrive, and the bytes are pulled here, so a socket reset
@@ -179,18 +232,24 @@ export const writeAtomic = async (
 // four attempts would mean sixteen requests for one file. The policy comes
 // from the client so a caller that configured one gets it here too.
 //
-// A retry starts the file over from byte zero: the secretstream pull state is
-// not resumable and there is no Range support on these endpoints.
+// Because the plaintext is streamed to disk rather than buffered, the atomic
+// write is part of the retried unit. A retry starts the file over from byte
+// zero — the secretstream pull state is not resumable and there is no Range
+// support — staging into a fresh temp file each time: a failed attempt writes
+// and then removes its own temp file, and only the attempt that reaches
+// TAG_FINAL renames one into place, so a download that needed three tries still
+// performs exactly one rename over the destination.
 const fetchAndDecrypt = async (
     api: ApiClient,
     openStream: () => Promise<ReadableStream<Uint8Array>>,
     header: Uint8Array,
     key: Uint8Array,
+    destination: string,
     onProgress?: ProgressCallback,
-): Promise<Uint8Array> =>
+): Promise<number> =>
     withRetry(async () => {
         const stream = await openStream();
-        return streamDecrypt(stream, header, key, onProgress);
+        return decryptToTemp(destination, stream, header, key, onProgress);
     }, api.getRetryOptions());
 
 export const downloadFile = async (
@@ -201,19 +260,15 @@ export const downloadFile = async (
 ): Promise<DownloadResult> => {
     const resolvedPath = outPath ?? file.metadata.title;
     const header = fromBase64(file.file.decryptionHeader);
-    const plaintext = await fetchAndDecrypt(
+    const bytesWritten = await fetchAndDecrypt(
         api,
         () => api.getFileStream(file.id, { retry: false }),
         header,
         file.key,
+        resolvedPath,
         onProgress,
     );
-    // Outside the retry, deliberately: only the attempt that produced a
-    // complete, authenticated plaintext gets to stage a temporary file, so a
-    // download that needed three tries still performs exactly one write and
-    // one rename.
-    await writeAtomic(resolvedPath, plaintext);
-    return { path: resolvedPath, bytesWritten: plaintext.length };
+    return { path: resolvedPath, bytesWritten };
 };
 
 export const downloadThumbnail = async (
@@ -224,13 +279,13 @@ export const downloadThumbnail = async (
 ): Promise<DownloadResult> => {
     const resolvedPath = outPath ?? `thumb_${file.metadata.title}`;
     const header = fromBase64(file.thumbnail.decryptionHeader);
-    const plaintext = await fetchAndDecrypt(
+    const bytesWritten = await fetchAndDecrypt(
         api,
         () => api.getThumbnailStream(file.id, { retry: false }),
         header,
         file.key,
+        resolvedPath,
         onProgress,
     );
-    await writeAtomic(resolvedPath, plaintext);
-    return { path: resolvedPath, bytesWritten: plaintext.length };
+    return { path: resolvedPath, bytesWritten };
 };

src/library/store.ts

@@ -0,0 +1,188 @@
+// On-disk JSON metadata store for the local cache.
+//
+// The store keeps one `metadata.json` file holding the account's server
+// state: the user id, a schema version, the cursor for the incremental
+// collections listing, and the decrypted collection and file records. The
+// whole file is read into RAM on load and rewritten as a whole on save; there
+// is no partial update and no lock file. A separate refresh unit populates the
+// store from the server — this module only stores what it is given.
+//
+// The file is a cache, so it is never trusted to exist or to be intact: a
+// missing or unreadable file loads as an empty store rather than an error, and
+// the refresh unit then repopulates it.
+
+import { mkdir, chmod, readFile } from "node:fs/promises";
+import { dirname } from "node:path";
+
+import { writeAtomic } from "../download/index.js";
+import type { Collection, EnteFile, Microseconds } from "../model/types.js";
+
+// Bumped only when the on-disk shape changes incompatibly. A file written
+// under a different version is discarded on load (see `load`): re-fetching
+// from the server is always safe and cheaper than migrating a cache.
+export const METADATA_SCHEMA_VERSION = 1;
+
+// Directory and file modes match `session.json`: the records hold decrypted
+// key material, so on a shared machine only the owner may read them.
+const DIR_MODE = 0o700;
+const FILE_MODE = 0o600;
+
+// On-disk shapes. They mirror the in-memory model exactly except for the
+// binary `key`, which JSON cannot hold and which is stored as base64.
+type StoredCollection = Omit<Collection, "key"> & { key: string };
+type StoredFile = Omit<EnteFile, "key"> & { key: string };
+
+interface StoredMetadata {
+    schemaVersion: number;
+    userID: number;
+    collectionsSinceTime: Microseconds;
+    collections: StoredCollection[];
+    files: StoredFile[];
+}
+
+const encodeKey = (key: Uint8Array): string =>
+    Buffer.from(key).toString("base64");
+
+const decodeKey = (encoded: string): Uint8Array =>
+    new Uint8Array(Buffer.from(encoded, "base64"));
+
+// A file membership is identified by the pair (collectionID, fileID): the same
+// underlying file can belong to several collections, each a distinct record
+// with its own key.
+const fileKey = (collectionID: number, fileID: number): string =>
+    `${collectionID}:${fileID}`;
+
+export class MetadataStore {
+    readonly path: string;
+    readonly schemaVersion = METADATA_SCHEMA_VERSION;
+    userID = 0;
+    collectionsSinceTime: Microseconds = 0;
+
+    private readonly collections = new Map<number, Collection>();
+    private readonly files = new Map<string, EnteFile>();
+
+    private constructor(path: string) {
+        this.path = path;
+    }
+
+    // Load the store at `path`. A missing file, an unreadable one, unparseable
+    // contents, or a mismatched schema version all yield an empty store bound
+    // to that path — never a thrown error, because the file is only a cache.
+    static async load(path: string): Promise<MetadataStore> {
+        const store = new MetadataStore(path);
+
+        let raw: string;
+        try {
+            raw = await readFile(path, "utf8");
+        } catch {
+            return store;
+        }
+
+        try {
+            const parsed = JSON.parse(raw) as StoredMetadata;
+            if (parsed.schemaVersion !== METADATA_SCHEMA_VERSION) {
+                return store;
+            }
+            store.userID = parsed.userID ?? 0;
+            store.collectionsSinceTime = parsed.collectionsSinceTime ?? 0;
+            for (const stored of parsed.collections ?? []) {
+                const collection: Collection = {
+                    ...stored,
+                    key: decodeKey(stored.key),
+                };
+                store.collections.set(collection.id, collection);
+            }
+            for (const stored of parsed.files ?? []) {
+                const file: EnteFile = {
+                    ...stored,
+                    key: decodeKey(stored.key),
+                };
+                store.files.set(fileKey(file.collectionID, file.id), file);
+            }
+        } catch {
+            // Any corruption discards the partial result: a half-read cache is
+            // worse than an empty one, since the refresh unit will rebuild it.
+            return new MetadataStore(path);
+        }
+
+        return store;
+    }
+
+    // Rewrite the whole file. The directory is created 0700 and the file left
+    // 0600; the write itself is the download layer's durable atomic writer
+    // (temp file, fsync, rename, dir fsync), so a reader never sees a partial
+    // file and a crash cannot leave a truncated one. There is no lock file and
+    // no `sync()` beyond the writer's own fsyncs.
+    async save(): Promise<void> {
+        const model: StoredMetadata = {
+            schemaVersion: METADATA_SCHEMA_VERSION,
+            userID: this.userID,
+            collectionsSinceTime: this.collectionsSinceTime,
+            collections: [...this.collections.values()].map((c) => ({
+                ...c,
+                key: encodeKey(c.key),
+            })),
+            files: [...this.files.values()].map((f) => ({
+                ...f,
+                key: encodeKey(f.key),
+            })),
+        };
+
+        const dir = dirname(this.path);
+        // chmod after mkdir so the mode is 0700 even when the directory
+        // already existed with a looser mode; mkdir alone would not tighten
+        // an existing directory.
+        await mkdir(dir, { recursive: true, mode: DIR_MODE });
+        await chmod(dir, DIR_MODE);
+
+        const payload = new TextEncoder().encode(
+            JSON.stringify(model, null, 2),
+        );
+        await writeAtomic(this.path, payload);
+        // The atomic writer's temp file inherits the default mode; tighten the
+        // renamed file to 0600. The 0700 directory already keeps other users
+        // out during the brief window before this runs.
+        await chmod(this.path, FILE_MODE);
+    }
+
+    getCollection(id: number): Collection | undefined {
+        return this.collections.get(id);
+    }
+
+    listCollections(): Collection[] {
+        return [...this.collections.values()];
+    }
+
+    putCollection(collection: Collection): void {
+        this.collections.set(collection.id, collection);
+    }
+
+    // Removing a collection also drops its file memberships: a file record is
+    // only meaningful as part of a collection the cache still knows about.
+    deleteCollection(id: number): void {
+        this.collections.delete(id);
+        for (const [key, file] of this.files) {
+            if (file.collectionID === id) {
+                this.files.delete(key);
+            }
+        }
+    }
+
+    getFile(collectionID: number, fileID: number): EnteFile | undefined {
+        return this.files.get(fileKey(collectionID, fileID));
+    }
+
+    listFiles(collectionID: number): EnteFile[] {
+        return [...this.files.values()].filter(
+            (f) => f.collectionID === collectionID,
+        );
+    }
+
+    putFile(file: EnteFile): void {
+        this.files.set(fileKey(file.collectionID, file.id), file);
+    }
+
+    deleteFile(collectionID: number, fileID: number): void {
+        this.files.delete(fileKey(collectionID, fileID));
+    }
+}

test/download/download.test.ts

@@ -118,6 +118,21 @@ const durabilityHook = vi.hoisted(() => ({
     events: [] as string[],
 }));
 
+/**
+ * `FileHandle.write` is wrapped so the tests can watch the streaming decrypt
+ * path put plaintext on disk one chunk at a time. This is the direct evidence
+ * that memory is bounded by the chunk size and not the file size: a buffered
+ * downloader would hand the whole file to a single write, whereas the streaming
+ * one issues one write per secretstream chunk, none larger than
+ * `STREAM_CHUNK_SIZE`. Each write records the temp path it targeted and its
+ * length. `writeFile` (which the whole-buffer `writeAtomic` uses) is a distinct
+ * native call and does not go through this method, so only the streaming path
+ * is observed here.
+ */
+const writeHook = vi.hoisted(() => ({
+    writes: [] as { path: string; length: number }[],
+}));
+
 vi.mock("node:fs/promises", async (importOriginal) => {
     const actual = await importOriginal<typeof import("node:fs/promises")>();
     const { existsSync: sourceExists } = await import("node:fs");
@@ -138,6 +153,22 @@ vi.mock("node:fs/promises", async (importOriginal) => {
                 durabilityHook.events.push(`sync:${String(flags)}:${path}`);
                 await realSync();
             };
+            const realWrite = handle.write.bind(handle);
+            handle.write = (async (
+                data: unknown,
+                ...rest2: unknown[]
+            ): Promise<unknown> => {
+                if (data instanceof Uint8Array) {
+                    writeHook.writes.push({
+                        path: String(path),
+                        length: data.length,
+                    });
+                }
+                return (realWrite as (...a: unknown[]) => Promise<unknown>)(
+                    data,
+                    ...rest2,
+                );
+            }) as typeof handle.write;
             return handle;
         },
         rename: async (from: string, to: string): Promise<void> => {
@@ -159,6 +190,7 @@ beforeEach(() => {
     renameHook.calls.length = 0;
     renameHook.failWith = null;
     durabilityHook.events.length = 0;
+    writeHook.writes.length = 0;
 });
 
 let testDir: string;
@@ -947,10 +979,13 @@ describe.each(entryPoints)("$name retries", ({ name, download }) => {
     });
 
     it("stages one temp file for the attempt that succeeded, not one per attempt", async () => {
-        // The atomic write stays outside the retry loop. A retried download
-        // must not leave a trail of half-written scratch files, and the
-        // destination must be touched exactly once — by the attempt that
-        // produced a complete, authenticated plaintext.
+        // Each streaming attempt stages into its own temp file, but a retried
+        // download must not leave a trail of half-written scratch files: a
+        // failed attempt removes its temp file, and the destination is renamed
+        // into place exactly once — by the attempt that produced a complete,
+        // authenticated plaintext. (Here the two failed attempts reset before a
+        // whole chunk is pulled, so they write nothing; the point stands either
+        // way — see the retry-restart test below, where they do write.)
         const { key, header, ciphertext } = smallFixture(42);
         const { fetch } = scriptedCdnFetch(
             { kind: "reset", bytes: ciphertext.slice(0, 16) },
@@ -1064,6 +1099,99 @@ describe.each(entryPoints)("$name retries", ({ name, download }) => {
     });
 });
 
+// ---------------------------------------------------------------------------
+// Streaming decrypt to disk
+//
+// The plaintext is never held whole in memory: each secretstream chunk is
+// written to the temp file as it is decrypted, so peak memory is bounded by the
+// chunk size rather than the file size. These tests watch the writes directly
+// (see `writeHook`) rather than infer memory behaviour from the final file.
+// ---------------------------------------------------------------------------
+
+describe.each(entryPoints)("$name streams to disk", ({ name, download }) => {
+    const freshDir = (): string =>
+        mkdtempSync(join(testDir, `${name}-stream-`));
+
+    /** Writes recorded against staged temp files (not the `writeFile` path). */
+    const tempWrites = (): { path: string; length: number }[] =>
+        writeHook.writes.filter((w) => w.path.endsWith(".tmp"));
+
+    it("writes one chunk at a time, none larger than STREAM_CHUNK_SIZE", async () => {
+        // The multi-chunk fixture decrypts to one full 4 MiB chunk plus a small
+        // final chunk. A streaming writer therefore issues exactly two writes,
+        // of STREAM_CHUNK_SIZE and then the final chunk's length — never a
+        // single write carrying the whole 4 MiB + 1 KiB file. That per-chunk
+        // shape is what "memory bounded by chunk size" means in practice: the
+        // plaintext is handed to the filesystem and dropped, chunk by chunk.
+        const { api, file } = fixtureFor(
+            multiChunkKey,
+            multiChunk.header,
+            multiChunk.body,
+        );
+        const outPath = join(freshDir(), "streamed.bin");
+
+        const result = await download(api, file, outPath);
+
+        const writes = tempWrites();
+        expect(writes.map((w) => w.length)).toEqual([
+            STREAM_CHUNK_SIZE,
+            multiChunk.plaintext.length - STREAM_CHUNK_SIZE,
+        ]);
+        // No single write ever carried the whole file, and every write fits in
+        // one chunk's worth of memory.
+        for (const w of writes) {
+            expect(w.length).toBeLessThanOrEqual(STREAM_CHUNK_SIZE);
+        }
+        expect(result.bytesWritten).toBe(multiChunk.plaintext.length);
+        expectSameBytes(readFileSync(outPath), multiChunk.plaintext);
+    });
+
+    it("restarts from byte zero on a retry, replacing the temp file cleanly", async () => {
+        // The secretstream pull state is not resumable, so a retry cannot
+        // continue a half-written file — it must start over. The first attempt
+        // here delivers a complete leading chunk and then stops before the
+        // TAG_FINAL chunk: 4 MiB of plaintext lands in a temp file, then the
+        // download is rejected as truncated and that temp file is discarded.
+        // The retry streams the whole body into a *fresh* temp file, so the
+        // destination ends up with exactly the plaintext once — never the
+        // leading chunk twice, and never a stale temp file left behind.
+        const truncatedBody = multiChunk.body.slice(
+            0,
+            multiChunk.finalChunkOffset,
+        );
+        const { fetch, requests } = scriptedCdnFetch(
+            { kind: "body", bytes: truncatedBody },
+            { kind: "body", bytes: multiChunk.body },
+        );
+        const api = new ApiClient({
+            fetch,
+            retry: { ...noWait, attempts: 4 },
+        });
+        const file = buildMockEnteFile(
+            multiChunkKey,
+            multiChunk.header,
+            multiChunk.header,
+        );
+        const dir = freshDir();
+        const outPath = join(dir, "retry-restart.bin");
+
+        const result = await download(api, file, outPath);
+
+        expect(requests()).toBe(2);
+        // Both attempts streamed to disk, each into its own temp file: the
+        // truncated first attempt wrote before it failed, proving the retry did
+        // not resume a partial file but replaced it.
+        const distinctTemps = new Set(tempWrites().map((w) => w.path));
+        expect(distinctTemps.size).toBe(2);
+        // The destination holds the complete plaintext exactly once, and no
+        // temp file survives.
+        expect(result.bytesWritten).toBe(multiChunk.plaintext.length);
+        expectSameBytes(readFileSync(outPath), multiChunk.plaintext);
+        expect(renameHook.calls).toHaveLength(1);
+        expect(readdirSync(dir)).toEqual(["retry-restart.bin"]);
+    });
+});
+
 describe("download retries: corruption is not retried", () => {
     it("gives up immediately on a chunk that failed to authenticate", async () => {
         // A whole chunk that failed to authenticate while the stream

test/library/store.test.ts

@@ -0,0 +1,246 @@
+/**
+ * Tests for the on-disk JSON metadata store (`MetadataStore`).
+ *
+ * The store is the local cache the library keeps of the account's server
+ * state: one `metadata.json` file holding the user id, a schema version, the
+ * cursor for the incremental collections listing, and the decrypted
+ * collection and file records. The whole file is read into RAM on load and
+ * rewritten as a whole on save. A separate refresh unit (issue #42) is what
+ * populates it; this unit only stores.
+ *
+ * Four contracts are load-bearing and each is exercised below:
+ *
+ * 1. **Round-trip fidelity.** Everything put into the store — including the
+ *    binary decryption keys, which JSON cannot hold directly and which the
+ *    store base64-encodes — comes back byte-for-byte after a save and a fresh
+ *    load. A cache that quietly dropped or mangled a field would hand the
+ *    caller wrong keys or stale metadata.
+ *
+ * 2. **A missing or corrupt file loads as an empty store, never an error.**
+ *    The file is only a cache: if it is absent (first run) or unreadable
+ *    (interrupted write on an older build, disk corruption, hand-editing),
+ *    the right answer is to start empty and let the refresh unit repopulate,
+ *    not to crash the whole library.
+ *
+ * 3. **Writes are atomic and durable.** The store reuses the same
+ *    fsync-before-rename atomic writer the download layer uses, so a reader
+ *    never sees a half-written file and a crash cannot leave a truncated one.
+ *    The observable consequence tested here is that a save leaves exactly the
+ *    destination file behind — no temporary sibling — and that overwriting an
+ *    existing store preserves a complete, re-loadable file.
+ *
+ * 4. **Permissions match `session.json`.** The directory is `0700` and the
+ *    file is `0600`, because the records contain decrypted key material and
+ *    must not be readable by other users on a shared machine.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import {
+    mkdtempSync,
+    rmSync,
+    readdirSync,
+    statSync,
+    writeFileSync,
+    mkdirSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import {
+    MetadataStore,
+    METADATA_SCHEMA_VERSION,
+} from "../../src/library/store.js";
+import type { Collection, EnteFile } from "../../src/model/types.js";
+
+// A representative decrypted collection, including a binary key and all three
+// magic-metadata layers, so the round-trip test proves every field survives.
+const sampleCollection = (): Collection => ({
+    id: 12345,
+    ownerID: 42,
+    key: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]),
+    name: "Holiday 2026",
+    type: "album",
+    updationTime: 1_700_000_000_000_000,
+    isShared: true,
+    magicMetadata: { visibility: 0 },
+    pubMagicMetadata: { subType: 0, coverID: 999 },
+    sharedMagicMetadata: { note: "shared with a friend" },
+});
+
+// A representative decrypted file membership: metadata, both blob headers, a
+// binary key, a content hash, and file/thumbnail sizes.
+const sampleFile = (): EnteFile => ({
+    id: 67890,
+    collectionID: 12345,
+    ownerID: 42,
+    key: new Uint8Array([9, 8, 7, 6, 5, 4, 3, 2, 1]),
+    metadata: {
+        title: "IMG_0001.jpg",
+        fileType: "image",
+        creationTime: 1_699_000_000_000_000,
+        modificationTime: 1_699_000_500_000_000,
+        latitude: 52.52,
+        longitude: 13.405,
+        hash: "sha256:deadbeef",
+    },
+    magicMetadata: { editedName: "sunset" },
+    pubMagicMetadata: { editedTime: 1_699_000_600_000_000 },
+    file: { decryptionHeader: "ZmlsZUhlYWRlcg==", size: 4_194_304 },
+    thumbnail: { decryptionHeader: "dGh1bWJIZWFkZXI=", size: 8192 },
+    updationTime: 1_700_000_100_000_000,
+});
+
+describe("MetadataStore", () => {
+    let dir: string;
+    let path: string;
+
+    beforeEach(() => {
+        dir = mkdtempSync(join(tmpdir(), "quak-store-"));
+        // Deliberately nest the store one level below the temp dir so save()
+        // has to create its own directory and set its mode.
+        path = join(dir, "cache", "metadata.json");
+    });
+
+    afterEach(() => {
+        rmSync(dir, { recursive: true, force: true });
+    });
+
+    it("round-trips the whole model, keys and all", async () => {
+        const store = await MetadataStore.load(path);
+        store.userID = 42;
+        store.collectionsSinceTime = 1_700_000_000_000_000;
+        store.putCollection(sampleCollection());
+        store.putFile(sampleFile());
+        await store.save();
+
+        const reloaded = await MetadataStore.load(path);
+        expect(reloaded.userID).toBe(42);
+        expect(reloaded.collectionsSinceTime).toBe(1_700_000_000_000_000);
+
+        // The binary key must come back as the exact bytes, not a base64
+        // string or a plain object of numbered keys.
+        const collection = reloaded.getCollection(12345);
+        expect(collection).toEqual(sampleCollection());
+        expect(collection?.key).toBeInstanceOf(Uint8Array);
+
+        const file = reloaded.getFile(12345, 67890);
+        expect(file).toEqual(sampleFile());
+        expect(file?.key).toBeInstanceOf(Uint8Array);
+
+        expect(reloaded.listCollections()).toHaveLength(1);
+        expect(reloaded.listFiles(12345)).toHaveLength(1);
+    });
+
+    it("writes the declared schema version", async () => {
+        const store = await MetadataStore.load(path);
+        await store.save();
+        const reloaded = await MetadataStore.load(path);
+        expect(reloaded.schemaVersion).toBe(METADATA_SCHEMA_VERSION);
+    });
+
+    it("loads an empty store when the file is missing", async () => {
+        const store = await MetadataStore.load(path);
+        expect(store.userID).toBe(0);
+        expect(store.listCollections()).toEqual([]);
+        expect(store.getCollection(1)).toBeUndefined();
+    });
+
+    it("loads an empty store when the file is corrupt", async () => {
+        mkdirSync(join(dir, "cache"), { recursive: true });
+        writeFileSync(path, "{ this is not valid json ][");
+        const store = await MetadataStore.load(path);
+        expect(store.listCollections()).toEqual([]);
+        expect(store.listFiles(12345)).toEqual([]);
+    });
+
+    it("loads an empty store when the schema version does not match", async () => {
+        // A cache written by a future build with an incompatible schema is
+        // discarded rather than misread; the refresh unit repopulates it.
+        mkdirSync(join(dir, "cache"), { recursive: true });
+        writeFileSync(
+            path,
+            JSON.stringify({
+                schemaVersion: METADATA_SCHEMA_VERSION + 1,
+                userID: 42,
+                collectionsSinceTime: 0,
+                collections: [],
+                files: [],
+            }),
+        );
+        const store = await MetadataStore.load(path);
+        expect(store.userID).toBe(0);
+        expect(store.listCollections()).toEqual([]);
+    });
+
+    it("creates the directory 0700 and the file 0600", async () => {
+        const store = await MetadataStore.load(path);
+        store.putCollection(sampleCollection());
+        await store.save();
+
+        // Directory 0700, file 0600: on a shared machine the decrypted keys
+        // in this file must be readable only by their owner. Mask to the
+        // permission bits; the file-type bits are not part of the assertion.
+        expect(statSync(join(dir, "cache")).mode & 0o777).toBe(0o700);
+        expect(statSync(path).mode & 0o777).toBe(0o600);
+    });
+
+    it("leaves exactly the destination behind, with no temp sibling", async () => {
+        const store = await MetadataStore.load(path);
+        store.putCollection(sampleCollection());
+        await store.save();
+        // The atomic writer stages a temporary file and renames it into
+        // place; on success nothing temporary is left in the directory.
+        expect(readdirSync(join(dir, "cache"))).toEqual(["metadata.json"]);
+    });
+
+    it("overwrites an existing store atomically and stays re-loadable", async () => {
+        const first = await MetadataStore.load(path);
+        first.userID = 1;
+        first.putCollection(sampleCollection());
+        await first.save();
+
+        const second = await MetadataStore.load(path);
+        second.userID = 2;
+        second.deleteCollection(12345);
+        await second.save();
+
+        const reloaded = await MetadataStore.load(path);
+        expect(reloaded.userID).toBe(2);
+        expect(reloaded.getCollection(12345)).toBeUndefined();
+        expect(readdirSync(join(dir, "cache"))).toEqual(["metadata.json"]);
+    });
+
+    it("deletes a collection together with its file memberships", async () => {
+        const store = await MetadataStore.load(path);
+        store.putCollection(sampleCollection());
+        store.putFile(sampleFile());
+        store.deleteCollection(12345);
+        expect(store.getCollection(12345)).toBeUndefined();
+        expect(store.getFile(12345, 67890)).toBeUndefined();
+        expect(store.listFiles(12345)).toEqual([]);
+    });
+
+    it("scopes file records to their collection membership", async () => {
+        // The same underlying file can be a member of two collections, each a
+        // separate record with its own key. Storing one must not touch the
+        // other, and lookups are per membership.
+        const store = await MetadataStore.load(path);
+        const inA = sampleFile();
+        const inB: EnteFile = {
+            ...sampleFile(),
+            collectionID: 55555,
+            key: new Uint8Array([100, 101, 102]),
+        };
+        store.putFile(inA);
+        store.putFile(inB);
+
+        expect(store.getFile(12345, 67890)?.key).toEqual(inA.key);
+        expect(store.getFile(55555, 67890)?.key).toEqual(inB.key);
+        expect(store.listFiles(12345)).toHaveLength(1);
+        expect(store.listFiles(55555)).toHaveLength(1);
+
+        store.deleteFile(12345, 67890);
+        expect(store.getFile(12345, 67890)).toBeUndefined();
+        expect(store.getFile(55555, 67890)?.key).toEqual(inB.key);
+    });
+});