Check downloaded originals against their recorded content hash (closes #68)

downloadFile, shared by quak get, the content cache and backup, hashes
the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and stores
nothing on a mismatch, failing with an error naming the file ID. A live
photo ZIP is unpacked with fflate and its image and video hashed
separately as <imageHash>:<videoHash>. decryptFile reads the older
imageHash/videoHash fields for live photos. A file with no recorded
hash is stored unchecked.

Model: opus-5-5

TODO.md

@@ -22,10 +22,10 @@ Tag v1.0.0.
   (issue 68). `downloadFile`, which `quak get`, the content cache and backup all
   use, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and
   stores nothing on a mismatch, failing with an error naming the file ID. A live
-  photo ZIP is unpacked as it streams with `fflate` and its image and video
-  hashed separately as `<imageHash>:<videoHash>`. `decryptFile` reads older
-  clients' `imageHash` and `videoHash` fields for live photos. A file with no
-  recorded hash is stored unchecked.
+  photo ZIP is unpacked with `fflate` and its image and video hashed separately
+  as `<imageHash>:<videoHash>`. `decryptFile` reads older clients' `imageHash`
+  and `videoHash` fields for live photos. A file with no recorded hash is stored
+  unchecked.
 - 2026-09-23: Single-sourced the version string (issue 5). `package.json` is the
   only place it is written: `src/index.ts` imports it for `VERSION` and
   `bin/quak.ts` passes `VERSION` to commander. tsc copies `package.json` to

src/download/index.ts

@@ -1,8 +1,8 @@
 import { randomUUID } from "node:crypto";
-import { open, rename, rm } from "node:fs/promises";
+import { open, readFile, rename, rm } from "node:fs/promises";
 import type { FileHandle } from "node:fs/promises";
 import { dirname, join } from "node:path";
-import { Unzip, UnzipInflate } from "fflate";
+import { unzipSync } from "fflate";
 import {
     chunkHashFinal,
     chunkHashInit,
@@ -181,7 +181,8 @@ export const fsyncPath = async (path: string): Promise<void> => {
 };
 
 // Stage a write to `destination` atomically and durably, then rename it into
-// place. `fill` writes the contents into the open temp file handle — either the
+// place. `fill` writes the contents into the open temp file handle (whose path
+// it is also given, so it can read back what it wrote) — either the
 // whole buffer at once (`writeAtomic`) or chunk by chunk as they decrypt
 // (`decryptToTemp`). The temp file is a sibling of the destination (same
 // directory, so the rename cannot cross a filesystem boundary), so callers
@@ -201,7 +202,7 @@ export const fsyncPath = async (path: string): Promise<void> => {
 // scratch file is left to fill the disk on repeated failures.
 const stageAtomic = async (
     destination: string,
-    fill: (handle: FileHandle) => Promise<void>,
+    fill: (handle: FileHandle, tmpPath: string) => Promise<void>,
 ): Promise<void> => {
     const dir = dirname(destination);
     // The random suffix keeps concurrent downloads of the same destination
@@ -210,7 +211,7 @@ const stageAtomic = async (
     try {
         const handle = await open(tmpPath, "w");
         try {
-            await fill(handle);
+            await fill(handle, tmpPath);
             await handle.sync();
         } finally {
             await handle.close();
@@ -241,81 +242,36 @@ export const writeAtomic = async (
 ): Promise<void> =>
     stageAtomic(destination, (handle) => handle.writeFile(plaintext));
 
-// Hashes an original's bytes as they are decrypted, for comparison with the
-// hash its uploader recorded.
-interface ContentHasher {
-    update: (plaintext: Uint8Array) => void;
-    digest: () => string;
-}
-
-const fileHasher = (): ContentHasher => {
+const hashBytes = (bytes: Uint8Array): string => {
     const state = chunkHashInit();
-    return {
-        update: (plaintext) => chunkHashUpdate(state, plaintext),
-        digest: () => chunkHashFinal(state),
-    };
+    chunkHashUpdate(state, bytes);
+    return chunkHashFinal(state);
 };
 
 // A live photo is stored as a ZIP of its image and its video, and its recorded
 // hash is `<imageHash>:<videoHash>`, each over that part's own bytes. Like the
-// upstream client's decoder, this takes the first entries whose names start
-// with `image` and `video`.
-//
-// The ZIP is chosen by its uploader and may expand enormously, so entries are
-// hashed as they decompress and never held. fflate's `Unzip` inflates each
-// push in one piece, and deflate expands at most about 1000-fold, so the ZIP
-// is pushed in 4 KiB slices to keep each decompressed piece near 4 MiB, one
-// plaintext chunk. Every entry is started, even one that is not hashed,
-// because fflate keeps an unstarted entry's data in memory.
-const livePhotoHasher = (fileID: number): ContentHasher => {
-    const sliceSize = 4096;
-    const fail = (message: string, cause?: unknown): Error =>
-        new Error(`download: file ${fileID}: ${message}`, { cause });
-    const claimed = new Set<string>();
-    const hashes = new Map<string, string>();
-    const unzip = new Unzip((entry) => {
-        const part = ["image", "video"].find((p) => entry.name.startsWith(p));
-        const target =
-            part === undefined || claimed.has(part)
-                ? undefined
-                : { part, state: chunkHashInit() };
-        if (target !== undefined) claimed.add(target.part);
-        entry.ondata = (err, data, final) => {
-            if (err) throw err;
-            if (target === undefined) return;
-            chunkHashUpdate(target.state, data);
-            if (final) hashes.set(target.part, chunkHashFinal(target.state));
-        };
-        entry.start();
-    });
-    unzip.register(UnzipInflate);
-    // fflate reports a bad ZIP by throwing, sometimes a TypeError, which the
-    // retry would take for a network failure; a bad ZIP is never retried.
-    const push = (data: Uint8Array, final: boolean): void => {
-        try {
-            unzip.push(data, final);
-        } catch (err) {
-            throw fail("live photo is not a readable ZIP", err);
-        }
-    };
-    return {
-        update: (plaintext) => {
-            for (let i = 0; i < plaintext.length; i += sliceSize) {
-                push(plaintext.subarray(i, i + sliceSize), false);
-            }
-        },
-        digest: () => {
-            push(new Uint8Array(0), true);
-            const image = hashes.get("image");
-            const video = hashes.get("video");
-            if (image === undefined || video === undefined) {
-                throw fail(
-                    "live photo ZIP does not hold both an image and a video",
-                );
-            }
-            return `${image}:${video}`;
-        },
-    };
+// upstream client's decoder, this takes the entries whose names start with
+// `image` and `video`. The ZIP is read whole: a live photo is a few megabytes.
+const livePhotoHash = async (
+    fileID: number,
+    zipPath: string,
+): Promise<string> => {
+    let entries: [string, Uint8Array][];
+    try {
+        entries = Object.entries(unzipSync(await readFile(zipPath)));
+    } catch (err) {
+        throw new Error(`download: file ${fileID}: live photo is not a ZIP`, {
+            cause: err,
+        });
+    }
+    const image = entries.find(([name]) => name.startsWith("image"));
+    const video = entries.find(([name]) => name.startsWith("video"));
+    if (image === undefined || video === undefined) {
+        throw new Error(
+            `download: file ${fileID}: live photo ZIP does not hold both an image and a video`,
+        );
+    }
+    return `${hashBytes(image[1])}:${hashBytes(video[1])}`;
 };
 
 // Decrypt `stream` straight to `destination`, one plaintext chunk at a time,
@@ -327,8 +283,8 @@ const livePhotoHasher = (fileID: number): ContentHasher => {
 //
 // `original` is the file whose original this is (none for a thumbnail, which
 // has no recorded hash). When its metadata has a hash, the decrypted bytes
-// must match it or nothing is stored. Both a plain file and a live photo's
-// parts are hashed as they stream. The mismatch error is not retried.
+// must match it or nothing is stored: a plain file is hashed as it streams, a
+// live photo from the finished ZIP. The mismatch error is not retried.
 const decryptToTemp = async (
     destination: string,
     stream: ReadableStream<Uint8Array>,
@@ -338,27 +294,26 @@ const decryptToTemp = async (
     original?: EnteFile,
 ): Promise<number> => {
     const expected = original?.metadata.hash;
-    const hasher =
-        original === undefined || expected === undefined
-            ? undefined
-            : original.metadata.fileType === "livePhoto"
-              ? livePhotoHasher(original.id)
-              : fileHasher();
+    const livePhoto = original?.metadata.fileType === "livePhoto";
+    const hashState =
+        expected !== undefined && !livePhoto ? chunkHashInit() : undefined;
     let bytesWritten = 0;
     try {
-        await stageAtomic(destination, async (handle) => {
+        await stageAtomic(destination, async (handle, tmpPath) => {
             bytesWritten = await streamDecrypt(
                 stream,
                 header,
                 key,
                 async (plaintext) => {
-                    hasher?.update(plaintext);
+                    if (hashState) chunkHashUpdate(hashState, plaintext);
                     await handle.write(plaintext);
                 },
                 onProgress,
             );
-            if (original === undefined || hasher === undefined) return;
-            const actual = hasher.digest();
+            if (original === undefined || expected === undefined) return;
+            const actual = hashState
+                ? chunkHashFinal(hashState)
+                : await livePhotoHash(original.id, tmpPath);
             if (actual !== expected) {
                 throw new Error(
                     `download: file ${original.id}: content hash ${actual} does not match the hash its uploader recorded, ${expected}`,

test/download/download.test.ts

@@ -189,31 +189,7 @@ vi.mock("node:fs/promises", async (importOriginal) => {
     };
 });
 
-/**
- * `chunkHashUpdate` is wrapped to record the length of every piece hashed, so
- * a test can show that a live photo entry reaches the hash in pieces far
- * smaller than the entry, rather than decompressed whole first.
- */
-const hashHook = vi.hoisted(() => ({
-    lengths: [] as number[],
-}));
-
-vi.mock("../../src/crypto/index.js", async (importOriginal) => {
-    const actual =
-        await importOriginal<typeof import("../../src/crypto/index.js")>();
-    return {
-        ...actual,
-        chunkHashUpdate: (
-            ...args: Parameters<typeof actual.chunkHashUpdate>
-        ): void => {
-            hashHook.lengths.push(args[1].length);
-            actual.chunkHashUpdate(...args);
-        },
-    };
-});
-
 beforeEach(() => {
-    hashHook.lengths.length = 0;
     renameHook.calls.length = 0;
     renameHook.failWith = null;
     durabilityHook.events.length = 0;
@@ -1713,27 +1689,6 @@ describe("downloadFile content hash", () => {
         expectSameBytes(readFileSync(t.outPath), livePhotoZip);
     });
 
-    it("hashes a large live photo entry as it decompresses, never whole", async () => {
-        // 64 MiB of zeros deflates to a few kilobytes, the shape of a ZIP
-        // that would exhaust memory if expanded whole.
-        const image = new Uint8Array(64 * 1024 * 1024);
-        const video = patternBytes(900, 83);
-        const zip = zipSync({ "image.heic": image, "video.mov": video });
-        const t = setup(zip, {
-            fileType: "livePhoto",
-            hash: `${blake2b(image)}:${blake2b(video)}`,
-        });
-
-        await t.run();
-
-        expectSameBytes(readFileSync(t.outPath), zip);
-        const hashed = hashHook.lengths.reduce((a, b) => a + b, 0);
-        expect(hashed).toBe(image.length + video.length);
-        expect(Math.max(...hashHook.lengths)).toBeLessThanOrEqual(
-            2 * STREAM_CHUNK_SIZE,
-        );
-    });
-
     it("rejects a live photo whose hash does not match", async () => {
         // The whole ZIP's hash is not the recorded one: each part is hashed.
         const t = setup(livePhotoZip, {

test/model/decrypt.test.ts

@@ -378,17 +378,6 @@ describe("model.decryptFile", () => {
         expect(
             hashOf({ fileType: 2, imageHash: "I", videoHash: 7 }),
         ).toBeUndefined();
-        // An empty string counts as absent, not as a hash to match.
-        expect(hashOf({ fileType: 0, hash: "" })).toBeUndefined();
-        expect(
-            hashOf({ fileType: 2, hash: "", imageHash: "I", videoHash: "V" }),
-        ).toBe("I:V");
-        expect(
-            hashOf({ fileType: 2, imageHash: "", videoHash: "V" }),
-        ).toBeUndefined();
-        expect(
-            hashOf({ fileType: 2, imageHash: "I", videoHash: "" }),
-        ).toBeUndefined();
     });
 
     it("maps fileType numbers to FileType strings", () => {