Check downloaded originals against their recorded content hash (closes #68)

downloadFile, shared by quak get, the content cache and backup, hashes
the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and stores
nothing on a mismatch, failing with an error naming the file ID. A live
photo ZIP is unpacked as it streams with fflate's Unzip, in small
slices so memory stays bounded however far an entry expands, and its
image and video hashed separately as <imageHash>:<videoHash>.
decryptFile reads the older imageHash/videoHash fields for live
photos. A file with no recorded hash is stored unchecked.

Model: opus-5-5

TODO.md

@@ -22,10 +22,10 @@ Tag v1.0.0.
   (issue 68). `downloadFile`, which `quak get`, the content cache and backup all
   use, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and
   stores nothing on a mismatch, failing with an error naming the file ID. A live
-  photo ZIP is unpacked with `fflate` and its image and video hashed separately
-  as `<imageHash>:<videoHash>`. `decryptFile` reads older clients' `imageHash`
-  and `videoHash` fields for live photos. A file with no recorded hash is stored
-  unchecked.
+  photo ZIP is unpacked as it streams with `fflate` and its image and video
+  hashed separately as `<imageHash>:<videoHash>`. `decryptFile` reads older
+  clients' `imageHash` and `videoHash` fields for live photos. A file with no
+  recorded hash is stored unchecked.
 - 2026-09-23: Single-sourced the version string (issue 5). `package.json` is the
   only place it is written: `src/index.ts` imports it for `VERSION` and
   `bin/quak.ts` passes `VERSION` to commander. tsc copies `package.json` to

src/download/index.ts

@@ -1,8 +1,8 @@
 import { randomUUID } from "node:crypto";
-import { open, readFile, rename, rm } from "node:fs/promises";
+import { open, rename, rm } from "node:fs/promises";
 import type { FileHandle } from "node:fs/promises";
 import { dirname, join } from "node:path";
-import { unzipSync } from "fflate";
+import { Unzip, UnzipInflate } from "fflate";
 import {
     chunkHashFinal,
     chunkHashInit,
@@ -181,8 +181,7 @@ export const fsyncPath = async (path: string): Promise<void> => {
 };
 
 // Stage a write to `destination` atomically and durably, then rename it into
-// place. `fill` writes the contents into the open temp file handle (whose path
-// it is also given, so it can read back what it wrote) — either the
+// place. `fill` writes the contents into the open temp file handle — either the
 // whole buffer at once (`writeAtomic`) or chunk by chunk as they decrypt
 // (`decryptToTemp`). The temp file is a sibling of the destination (same
 // directory, so the rename cannot cross a filesystem boundary), so callers
@@ -202,7 +201,7 @@ export const fsyncPath = async (path: string): Promise<void> => {
 // scratch file is left to fill the disk on repeated failures.
 const stageAtomic = async (
     destination: string,
-    fill: (handle: FileHandle, tmpPath: string) => Promise<void>,
+    fill: (handle: FileHandle) => Promise<void>,
 ): Promise<void> => {
     const dir = dirname(destination);
     // The random suffix keeps concurrent downloads of the same destination
@@ -211,7 +210,7 @@ const stageAtomic = async (
     try {
         const handle = await open(tmpPath, "w");
         try {
-            await fill(handle, tmpPath);
+            await fill(handle);
             await handle.sync();
         } finally {
             await handle.close();
@@ -242,36 +241,81 @@ export const writeAtomic = async (
 ): Promise<void> =>
     stageAtomic(destination, (handle) => handle.writeFile(plaintext));
 
-const hashBytes = (bytes: Uint8Array): string => {
+// Hashes an original's bytes as they are decrypted, for comparison with the
+// hash its uploader recorded.
+interface ContentHasher {
+    update: (plaintext: Uint8Array) => void;
+    digest: () => string;
+}
+
+const fileHasher = (): ContentHasher => {
     const state = chunkHashInit();
-    chunkHashUpdate(state, bytes);
-    return chunkHashFinal(state);
+    return {
+        update: (plaintext) => chunkHashUpdate(state, plaintext),
+        digest: () => chunkHashFinal(state),
+    };
 };
 
 // A live photo is stored as a ZIP of its image and its video, and its recorded
 // hash is `<imageHash>:<videoHash>`, each over that part's own bytes. Like the
-// upstream client's decoder, this takes the entries whose names start with
-// `image` and `video`. The ZIP is read whole: a live photo is a few megabytes.
-const livePhotoHash = async (
-    fileID: number,
-    zipPath: string,
-): Promise<string> => {
-    let entries: [string, Uint8Array][];
-    try {
-        entries = Object.entries(unzipSync(await readFile(zipPath)));
-    } catch (err) {
-        throw new Error(`download: file ${fileID}: live photo is not a ZIP`, {
-            cause: err,
-        });
-    }
-    const image = entries.find(([name]) => name.startsWith("image"));
-    const video = entries.find(([name]) => name.startsWith("video"));
-    if (image === undefined || video === undefined) {
-        throw new Error(
-            `download: file ${fileID}: live photo ZIP does not hold both an image and a video`,
-        );
-    }
-    return `${hashBytes(image[1])}:${hashBytes(video[1])}`;
+// upstream client's decoder, this takes the first entries whose names start
+// with `image` and `video`.
+//
+// The ZIP is chosen by its uploader and may expand enormously, so entries are
+// hashed as they decompress and never held. fflate's `Unzip` inflates each
+// push in one piece, and deflate expands at most about 1000-fold, so the ZIP
+// is pushed in 4 KiB slices to keep each decompressed piece near 4 MiB, one
+// plaintext chunk. Every entry is started, even one that is not hashed,
+// because fflate keeps an unstarted entry's data in memory.
+const livePhotoHasher = (fileID: number): ContentHasher => {
+    const sliceSize = 4096;
+    const fail = (message: string, cause?: unknown): Error =>
+        new Error(`download: file ${fileID}: ${message}`, { cause });
+    const claimed = new Set<string>();
+    const hashes = new Map<string, string>();
+    const unzip = new Unzip((entry) => {
+        const part = ["image", "video"].find((p) => entry.name.startsWith(p));
+        const target =
+            part === undefined || claimed.has(part)
+                ? undefined
+                : { part, state: chunkHashInit() };
+        if (target !== undefined) claimed.add(target.part);
+        entry.ondata = (err, data, final) => {
+            if (err) throw err;
+            if (target === undefined) return;
+            chunkHashUpdate(target.state, data);
+            if (final) hashes.set(target.part, chunkHashFinal(target.state));
+        };
+        entry.start();
+    });
+    unzip.register(UnzipInflate);
+    // fflate reports a bad ZIP by throwing, sometimes a TypeError, which the
+    // retry would take for a network failure; a bad ZIP is never retried.
+    const push = (data: Uint8Array, final: boolean): void => {
+        try {
+            unzip.push(data, final);
+        } catch (err) {
+            throw fail("live photo is not a readable ZIP", err);
+        }
+    };
+    return {
+        update: (plaintext) => {
+            for (let i = 0; i < plaintext.length; i += sliceSize) {
+                push(plaintext.subarray(i, i + sliceSize), false);
+            }
+        },
+        digest: () => {
+            push(new Uint8Array(0), true);
+            const image = hashes.get("image");
+            const video = hashes.get("video");
+            if (image === undefined || video === undefined) {
+                throw fail(
+                    "live photo ZIP does not hold both an image and a video",
+                );
+            }
+            return `${image}:${video}`;
+        },
+    };
 };
 
 // Decrypt `stream` straight to `destination`, one plaintext chunk at a time,
@@ -283,8 +327,8 @@ const livePhotoHash = async (
 //
 // `original` is the file whose original this is (none for a thumbnail, which
 // has no recorded hash). When its metadata has a hash, the decrypted bytes
-// must match it or nothing is stored: a plain file is hashed as it streams, a
-// live photo from the finished ZIP. The mismatch error is not retried.
+// must match it or nothing is stored. Both a plain file and a live photo's
+// parts are hashed as they stream. The mismatch error is not retried.
 const decryptToTemp = async (
     destination: string,
     stream: ReadableStream<Uint8Array>,
@@ -294,26 +338,27 @@ const decryptToTemp = async (
     original?: EnteFile,
 ): Promise<number> => {
     const expected = original?.metadata.hash;
-    const livePhoto = original?.metadata.fileType === "livePhoto";
-    const hashState =
-        expected !== undefined && !livePhoto ? chunkHashInit() : undefined;
+    const hasher =
+        original === undefined || expected === undefined
+            ? undefined
+            : original.metadata.fileType === "livePhoto"
+              ? livePhotoHasher(original.id)
+              : fileHasher();
     let bytesWritten = 0;
     try {
-        await stageAtomic(destination, async (handle, tmpPath) => {
+        await stageAtomic(destination, async (handle) => {
             bytesWritten = await streamDecrypt(
                 stream,
                 header,
                 key,
                 async (plaintext) => {
-                    if (hashState) chunkHashUpdate(hashState, plaintext);
+                    hasher?.update(plaintext);
                     await handle.write(plaintext);
                 },
                 onProgress,
             );
-            if (original === undefined || expected === undefined) return;
-            const actual = hashState
-                ? chunkHashFinal(hashState)
-                : await livePhotoHash(original.id, tmpPath);
+            if (original === undefined || hasher === undefined) return;
+            const actual = hasher.digest();
             if (actual !== expected) {
                 throw new Error(
                     `download: file ${original.id}: content hash ${actual} does not match the hash its uploader recorded, ${expected}`,

test/download/download.test.ts

@@ -189,7 +189,31 @@ vi.mock("node:fs/promises", async (importOriginal) => {
     };
 });
 
+/**
+ * `chunkHashUpdate` is wrapped to record the length of every piece hashed, so
+ * a test can show that a live photo entry reaches the hash in pieces far
+ * smaller than the entry, rather than decompressed whole first.
+ */
+const hashHook = vi.hoisted(() => ({
+    lengths: [] as number[],
+}));
+
+vi.mock("../../src/crypto/index.js", async (importOriginal) => {
+    const actual =
+        await importOriginal<typeof import("../../src/crypto/index.js")>();
+    return {
+        ...actual,
+        chunkHashUpdate: (
+            ...args: Parameters<typeof actual.chunkHashUpdate>
+        ): void => {
+            hashHook.lengths.push(args[1].length);
+            actual.chunkHashUpdate(...args);
+        },
+    };
+});
+
 beforeEach(() => {
+    hashHook.lengths.length = 0;
     renameHook.calls.length = 0;
     renameHook.failWith = null;
     durabilityHook.events.length = 0;
@@ -1689,6 +1713,27 @@ describe("downloadFile content hash", () => {
         expectSameBytes(readFileSync(t.outPath), livePhotoZip);
     });
 
+    it("hashes a large live photo entry as it decompresses, never whole", async () => {
+        // 64 MiB of zeros deflates to a few kilobytes, the shape of a ZIP
+        // that would exhaust memory if expanded whole.
+        const image = new Uint8Array(64 * 1024 * 1024);
+        const video = patternBytes(900, 83);
+        const zip = zipSync({ "image.heic": image, "video.mov": video });
+        const t = setup(zip, {
+            fileType: "livePhoto",
+            hash: `${blake2b(image)}:${blake2b(video)}`,
+        });
+
+        await t.run();
+
+        expectSameBytes(readFileSync(t.outPath), zip);
+        const hashed = hashHook.lengths.reduce((a, b) => a + b, 0);
+        expect(hashed).toBe(image.length + video.length);
+        expect(Math.max(...hashHook.lengths)).toBeLessThanOrEqual(
+            2 * STREAM_CHUNK_SIZE,
+        );
+    });
+
     it("rejects a live photo whose hash does not match", async () => {
         // The whole ZIP's hash is not the recorded one: each part is hashed.
         const t = setup(livePhotoZip, {

test/model/decrypt.test.ts

@@ -378,6 +378,17 @@ describe("model.decryptFile", () => {
         expect(
             hashOf({ fileType: 2, imageHash: "I", videoHash: 7 }),
         ).toBeUndefined();
+        // An empty string counts as absent, not as a hash to match.
+        expect(hashOf({ fileType: 0, hash: "" })).toBeUndefined();
+        expect(
+            hashOf({ fileType: 2, hash: "", imageHash: "I", videoHash: "V" }),
+        ).toBe("I:V");
+        expect(
+            hashOf({ fileType: 2, imageHash: "", videoHash: "V" }),
+        ).toBeUndefined();
+        expect(
+            hashOf({ fileType: 2, imageHash: "I", videoHash: "" }),
+        ).toBeUndefined();
     });
 
     it("maps fileType numbers to FileType strings", () => {