Check downloaded originals against their recorded content hash (closes #68)

downloadFile, shared by quak get, the content cache and backup, hashes
the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and stores
nothing on a mismatch, failing with an error naming the file ID. A live
photo ZIP is unpacked as it streams with fflate's Unzip, in small
slices so memory stays bounded however far an entry expands, and its
image and video hashed separately as <imageHash>:<videoHash>.
decryptFile reads the older imageHash/videoHash fields for live
photos. A file with no recorded hash is stored unchecked.

Model: opus-5-5

README.md

@@ -471,6 +471,11 @@ accepted for backward compatibility but ignored. `backup-metadata --exif` (alias
 metadata. The listing and backup commands support `--json` for machine-readable
 output.
 
+`backup-metadata` fetches ML data in requests of up to 200 files. When a request
+still fails after its retries, the error is logged, each of its files is written
+with the reason in an `mlDataError` field instead of `mlData`, and the dump goes
+on. The exit code is non-zero if any ML data request failed.
+
 `helper fix-missing-thumbnails` regenerates thumbnails for baseline JPEG images
 only, because the bundled decoder (`jpeg-js`) decodes only JPEG. A non-JPEG
 image (PNG, HEIC) or a video is reported as `skipped` (unsupported format), kept
@@ -677,10 +682,13 @@ Under `cacheDirectory`:
 ```
 
 A stored file appears only via an atomic temp-then-rename, so its presence means
-it is complete. The design also calls for a content-hash comparison against
+it is complete. Every downloaded original (by `quak get`, the cache, or
-`FileMetadata.hash` on each fetched original; that check is deferred (issue
+`backup`) whose metadata records a content hash (`FileMetadata.hash`) is hashed
-https://git.eeqj.de/sneak/quak/issues/68) because the exact hash construction
+as it is written: unkeyed BLAKE2b with a 64-byte output, standard base64. For a
-cannot yet be confirmed against the repo's fixtures.
+live photo, which is stored as a ZIP, the image and the video are hashed
+separately and joined as `<imageHash>:<videoHash>`. A mismatch stores nothing
+and fails the download with an error naming the file ID. An original with no
+recorded hash, from a very old client, is stored unchecked.
 
 ### Key types by source file
 

TODO.md

@@ -18,6 +18,20 @@ Tag v1.0.0.
 
 # Completed Steps
 
+- 2026-09-23: Checked downloaded originals against their recorded content hash
+  (issue 68). `downloadFile`, which `quak get`, the content cache and backup all
+  use, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and
+  stores nothing on a mismatch, failing with an error naming the file ID. A live
+  photo ZIP is unpacked as it streams with `fflate` and its image and video
+  hashed separately as `<imageHash>:<videoHash>`. `decryptFile` reads older
+  clients' `imageHash` and `videoHash` fields for live photos. A file with no
+  recorded hash is stored unchecked.
+- 2026-09-23: `backup-metadata` no longer stops on one failed ML data request
+  (issue 101). Each request of up to 200 files is tried on its own; a failed one
+  is logged, its files are written with the reason in `mlDataError`, and the
+  command exits 1 once the dump is complete. `fetchMLData`, which only this
+  command used, is gone; the command calls `fetchMLDataBatch` per batch.
+
 - 2026-09-23: Single-sourced the version string (issue 5). `package.json` is the
   only place it is written: `src/index.ts` imports it for `VERSION` and
   `bin/quak.ts` passes `VERSION` to commander. tsc copies `package.json` to

package.json

@@ -42,6 +42,7 @@
         "env-paths": "4.0.0",
         "exif-reader": "2.0.3",
         "fast-srp-hap": "2.0.4",
+        "fflate": "0.8.3",
         "jpeg-js": "0.4.4",
         "libsodium-wrappers-sumo": "0.8.4"
     }

src/cli-commands.ts

@@ -323,11 +323,11 @@ export const backupMetadataCommand = async (
     if (!client) return 1;
     const lib = await openReadLibrary(ctx, client);
     try {
-        await runMetadataBackup(lib, client, dir, {
+        const { failedMLBatches } = await runMetadataBackup(lib, client, dir, {
             exif: opts.exif || opts.all,
             onProgress: (msg) => ctx.stderr.write(msg + "\n"),
         });
-        return 0;
+        return failedMLBatches > 0 ? 1 : 0;
     } finally {
         await lib.close();
     }

src/crypto/hash.ts

@@ -0,0 +1,22 @@
+import sodium, { type StateAddress } from "libsodium-wrappers-sumo";
+import { toBase64 } from "./encoding.js";
+
+// The content hash an uploading client records in a file's metadata: unkeyed
+// BLAKE2b with a 64-byte output over the original's bytes, fed in chunks, as
+// standard base64 with padding. Named after the upstream client's functions.
+// The output length is read at call time for the same reason as
+// `streamTagFinal` in stream.ts: libsodium sets its constants only once ready.
+
+export const chunkHashInit = (): StateAddress =>
+    sodium.crypto_generichash_init(null, sodium.crypto_generichash_BYTES_MAX);
+
+export const chunkHashUpdate = (state: StateAddress, chunk: Uint8Array): void =>
+    sodium.crypto_generichash_update(state, chunk);
+
+export const chunkHashFinal = (state: StateAddress): string =>
+    toBase64(
+        sodium.crypto_generichash_final(
+            state,
+            sodium.crypto_generichash_BYTES_MAX,
+        ),
+    );

src/crypto/index.ts

@@ -7,6 +7,7 @@ export {
 } from "./encoding.js";
 export { deriveKEK, deriveLoginSubkey } from "./kdf.js";
 export { decryptBox, decryptSealed } from "./box.js";
+export { chunkHashFinal, chunkHashInit, chunkHashUpdate } from "./hash.js";
 export {
     decryptBlob,
     encryptBlob,

src/download/index.ts

@@ -2,7 +2,11 @@ import { randomUUID } from "node:crypto";
 import { open, rename, rm } from "node:fs/promises";
 import type { FileHandle } from "node:fs/promises";
 import { dirname, join } from "node:path";
+import { Unzip, UnzipInflate } from "fflate";
 import {
+    chunkHashFinal,
+    chunkHashInit,
+    chunkHashUpdate,
     fromBase64,
     initStreamPull,
     pullStreamChunk,
@@ -237,19 +241,109 @@ export const writeAtomic = async (
 ): Promise<void> =>
     stageAtomic(destination, (handle) => handle.writeFile(plaintext));
 
+// Hashes an original's bytes as they are decrypted, for comparison with the
+// hash its uploader recorded.
+interface ContentHasher {
+    update: (plaintext: Uint8Array) => void;
+    digest: () => string;
+}
+
+const fileHasher = (): ContentHasher => {
+    const state = chunkHashInit();
+    return {
+        update: (plaintext) => chunkHashUpdate(state, plaintext),
+        digest: () => chunkHashFinal(state),
+    };
+};
+
+// A live photo is stored as a ZIP of its image and its video, and its recorded
+// hash is `<imageHash>:<videoHash>`, each over that part's own bytes. Like the
+// upstream client's decoder, this takes the first entries whose names start
+// with `image` and `video`.
+//
+// The ZIP is chosen by its uploader and may expand enormously, so entries are
+// hashed as they decompress and never held. fflate's `Unzip` inflates each
+// push in one piece, and deflate expands at most about 1000-fold, so the ZIP
+// is pushed in 4 KiB slices to keep each decompressed piece near 4 MiB, one
+// plaintext chunk. Every entry is started, even one that is not hashed,
+// because fflate keeps an unstarted entry's data in memory.
+const livePhotoHasher = (fileID: number): ContentHasher => {
+    const sliceSize = 4096;
+    const fail = (message: string, cause?: unknown): Error =>
+        new Error(`download: file ${fileID}: ${message}`, { cause });
+    const claimed = new Set<string>();
+    const hashes = new Map<string, string>();
+    const unzip = new Unzip((entry) => {
+        const part = ["image", "video"].find((p) => entry.name.startsWith(p));
+        const target =
+            part === undefined || claimed.has(part)
+                ? undefined
+                : { part, state: chunkHashInit() };
+        if (target !== undefined) claimed.add(target.part);
+        entry.ondata = (err, data, final) => {
+            if (err) throw err;
+            if (target === undefined) return;
+            chunkHashUpdate(target.state, data);
+            if (final) hashes.set(target.part, chunkHashFinal(target.state));
+        };
+        entry.start();
+    });
+    unzip.register(UnzipInflate);
+    // fflate reports a bad ZIP by throwing, sometimes a TypeError, which the
+    // retry would take for a network failure; a bad ZIP is never retried.
+    const push = (data: Uint8Array, final: boolean): void => {
+        try {
+            unzip.push(data, final);
+        } catch (err) {
+            throw fail("live photo is not a readable ZIP", err);
+        }
+    };
+    return {
+        update: (plaintext) => {
+            for (let i = 0; i < plaintext.length; i += sliceSize) {
+                push(plaintext.subarray(i, i + sliceSize), false);
+            }
+        },
+        digest: () => {
+            push(new Uint8Array(0), true);
+            const image = hashes.get("image");
+            const video = hashes.get("video");
+            if (image === undefined || video === undefined) {
+                throw fail(
+                    "live photo ZIP does not hold both an image and a video",
+                );
+            }
+            return `${image}:${video}`;
+        },
+    };
+};
+
 // Decrypt `stream` straight to `destination`, one plaintext chunk at a time,
 // under the atomic writer's temp-then-rename discipline. Memory stays bounded
 // by the chunk size: each decrypted chunk is written to the temp file and
 // dropped. The rename happens only after the stream authenticates as terminated
 // on TAG_FINAL; a truncated stream throws and leaves the destination untouched.
 // Returns the plaintext length written.
+//
+// `original` is the file whose original this is (none for a thumbnail, which
+// has no recorded hash). When its metadata has a hash, the decrypted bytes
+// must match it or nothing is stored. Both a plain file and a live photo's
+// parts are hashed as they stream. The mismatch error is not retried.
 const decryptToTemp = async (
     destination: string,
     stream: ReadableStream<Uint8Array>,
     header: Uint8Array,
     key: Uint8Array,
     onProgress?: ProgressCallback,
+    original?: EnteFile,
 ): Promise<number> => {
+    const expected = original?.metadata.hash;
+    const hasher =
+        original === undefined || expected === undefined
+            ? undefined
+            : original.metadata.fileType === "livePhoto"
+              ? livePhotoHasher(original.id)
+              : fileHasher();
     let bytesWritten = 0;
     try {
         await stageAtomic(destination, async (handle) => {
@@ -258,10 +352,18 @@ const decryptToTemp = async (
                 header,
                 key,
                 async (plaintext) => {
+                    hasher?.update(plaintext);
                     await handle.write(plaintext);
                 },
                 onProgress,
             );
+            if (original === undefined || hasher === undefined) return;
+            const actual = hasher.digest();
+            if (actual !== expected) {
+                throw new Error(
+                    `download: file ${original.id}: content hash ${actual} does not match the hash its uploader recorded, ${expected}`,
+                );
+            }
         });
     } catch (err) {
         // Cancel the body so its connection is closed now rather than held
@@ -302,10 +404,18 @@ const fetchAndDecrypt = async (
     key: Uint8Array,
     destination: string,
     onProgress?: ProgressCallback,
+    original?: EnteFile,
 ): Promise<number> =>
     withRetry(async () => {
         const stream = await openStream();
-        return decryptToTemp(destination, stream, header, key, onProgress);
+        return decryptToTemp(
+            destination,
+            stream,
+            header,
+            key,
+            onProgress,
+            original,
+        );
     }, api.getRetryOptions());
 
 export const downloadFile = async (
@@ -326,6 +436,7 @@ export const downloadFile = async (
         file.key,
         resolvedPath,
         onProgress,
+        file,
     );
     return { path: resolvedPath, bytesWritten };
 };

src/library/content.ts

@@ -15,12 +15,12 @@
 // Integrity. The reused streaming decrypt is the enforced guarantee: every
 // chunk is authenticated and the writer renames the file into place only once
 // the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and
-// nothing is stored. On top of that this module refuses to record a stored file
+// nothing is stored. For an original whose metadata records a content hash
-// that came out empty. The design also asks for a content-hash comparison
+// (`FileMetadata.hash`), the writer also hashes the decrypted bytes and stores
-// against `FileMetadata.hash` (with a `fileSize` fallback); that is deferred —
+// nothing if they differ, failing the fetch with an error naming the file. An
-// see the PR — because the exact hash construction cannot be confirmed against
+// original with no recorded hash is stored unchecked, as the upstream client
-// the repo's fixtures and `FileBlob.size` is the encrypted object size, not the
+// does; thumbnails have none. On top of that this module refuses to record a
-// decrypted length this layer has.
+// stored file that came out empty.
 
 import { existsSync, statSync } from "node:fs";
 import {

src/metadata-backup.ts

@@ -5,7 +5,11 @@ import exifReader from "exif-reader";
 import type { Client } from "./client.js";
 import type { Library, Photo } from "./library/index.js";
 import { sanitizeFileName } from "./filename.js";
-import { fetchMLData } from "./mldata-fetch.js";
+import {
+    fetchMLDataBatch,
+    MLDATA_BATCH_SIZE,
+    type MLData,
+} from "./mldata-fetch.js";
 import type { EnteFile } from "./model/types.js";
 
 export type ProgressCallback = (message: string) => void;
@@ -137,13 +141,14 @@ const extractExif = async (
 // of plain JSON: account, per-collection, and per-file records including the
 // private and public magic metadata and (by default) the ML data. Collections
 // and files are enumerated from the library's cache rather than a fresh server
-// scan; the ML fetch and EXIF extraction are unchanged.
+// scan. Returns how many ML data requests failed; their files are still
+// written, with `mlDataError` in place of `mlData`.
 export const runMetadataBackup = async (
     lib: Library,
     client: Client,
     outDir: string,
     opts?: MetadataBackupOptions,
-): Promise<void> => {
+): Promise<{ failedMLBatches: number }> => {
     const log = opts?.onProgress ?? (() => {});
     const wantExif = opts?.exif ?? false;
 
@@ -208,12 +213,31 @@ export const runMetadataBackup = async (
         }
     }
 
+    // One failed request (retries exhausted) must not end the dump: its files
+    // get the reason in `mlDataError` and the other batches go on.
     log("Fetching ML data (face detections, CLIP embeddings)...");
-    const mlDataMap = await fetchMLData(
+    const mlDataMap = new Map<number, MLData>();
+    const mlDataErrors = new Map<number, string>();
+    let failedMLBatches = 0;
+    const fileIDs = [...fileKeys.keys()];
+    for (let i = 0; i < fileIDs.length; i += MLDATA_BATCH_SIZE) {
+        const batch = fileIDs.slice(i, i + MLDATA_BATCH_SIZE);
+        try {
+            const result = await fetchMLDataBatch(
                 client.getApiClient(),
-        [...fileKeys.keys()],
+                batch,
                 fileKeys,
             );
+            for (const [id, payload] of result) mlDataMap.set(id, payload);
+        } catch (err) {
+            const reason = err instanceof Error ? err.message : String(err);
+            failedMLBatches++;
+            log(
+                `ML data request for ${batch.length} file(s) failed: ${reason}`,
+            );
+            for (const id of batch) mlDataErrors.set(id, reason);
+        }
+    }
     log(`Got ML data for ${mlDataMap.size} file(s)`);
 
     const writtenFileIDs = new Set<number>();
@@ -233,6 +257,8 @@ export const runMetadataBackup = async (
 
         const ml = mlDataMap.get(file.id);
         if (ml) fileMeta.mlData = ml;
+        const mlError = mlDataErrors.get(file.id);
+        if (mlError) fileMeta.mlDataError = mlError;
 
         if (wantExif && !writtenFileIDs.has(file.id)) {
             log(`[${file.metadata.title}] Extracting EXIF...`);
@@ -253,4 +279,5 @@ export const runMetadataBackup = async (
     }
 
     log("Metadata backup complete.");
+    return { failedMLBatches };
 };

src/mldata-fetch.ts

@@ -5,9 +5,8 @@
 // comes back encrypted under the file's own key and gzipped; decrypting and
 // gunzipping yields the JSON payload
 // `{ face: { faces: [...] }, clip: { embedding } }`. Ente caps a request at 200
-// ids, so `fetchMLData` batches for callers that want many at once while
+// ids, so callers that want many at once split them into batches of
-// `fetchMLDataBatch` is the single-request unit the library submits to its
+// `MLDATA_BATCH_SIZE` and call `fetchMLDataBatch` once per batch.
-// request pool.
 
 import { gunzipSync } from "node:zlib";
 
@@ -69,25 +68,3 @@ export const fetchMLDataBatch = async (
     }
     return result;
 };
-
-// Fetch ML data for arbitrarily many ids, batching at `MLDATA_BATCH_SIZE`. Used
-// by the one-shot metadata backup; the library fetches through its request pool
-// with `fetchMLDataBatch` instead.
-export const fetchMLData = async (
-    api: ApiClient,
-    fileIDs: number[],
-    fileKeys: Map<number, Uint8Array>,
-): Promise<Map<number, MLData>> => {
-    const result = new Map<number, MLData>();
-    for (let i = 0; i < fileIDs.length; i += MLDATA_BATCH_SIZE) {
-        const batch = fileIDs.slice(i, i + MLDATA_BATCH_SIZE);
-        for (const [id, payload] of await fetchMLDataBatch(
-            api,
-            batch,
-            fileKeys,
-        )) {
-            result.set(id, payload);
-        }
-    }
-    return result;
-};

src/model/decrypt.ts

@@ -34,6 +34,28 @@ const FILE_TYPE_MAP: Record<number, FileType> = {
 
 const parseFileType = (n: number): FileType => FILE_TYPE_MAP[n] ?? "unknown";
 
+// The hash the uploading client recorded for the original's bytes, read the
+// way the upstream client's `metadataHash` reads it: `hash` if present,
+// otherwise, for a live photo from an older client that wrote the two parts
+// separately, `<imageHash>:<videoHash>`. A field that is not a non-empty
+// string counts as absent, and a file with no hash at all is normal.
+const expectedHash = (json: Record<string, unknown>): string | undefined => {
+    const text = (v: unknown): string | undefined =>
+        typeof v === "string" && v !== "" ? v : undefined;
+    const hash = text(json.hash);
+    if (hash !== undefined) return hash;
+    const imageHash = text(json.imageHash);
+    const videoHash = text(json.videoHash);
+    if (
+        json.fileType === 2 &&
+        imageHash !== undefined &&
+        videoHash !== undefined
+    ) {
+        return `${imageHash}:${videoHash}`;
+    }
+    return undefined;
+};
+
 export const decryptCollection = (
     raw: RawCollection,
     keys: KeyMaterial,
@@ -115,7 +137,7 @@ export const decryptFile = (
         modificationTime: metadataJSON.modificationTime ?? 0,
         latitude: metadataJSON.latitude,
         longitude: metadataJSON.longitude,
-        hash: metadataJSON.hash,
+        hash: expectedHash(metadataJSON),
     };
 
     const magicMetadata = decryptMagicMetadata(raw.magicMetadata, key);

src/model/types.ts

@@ -29,6 +29,9 @@ export interface FileMetadata {
     modificationTime: Microseconds;
     latitude?: number;
     longitude?: number;
+    // The content hash the uploader recorded (see `expectedHash` in
+    // decrypt.ts); `downloadFile` refuses an original that does not match it.
+    // Absent for files from very old clients.
     hash?: string;
 }
 

test/cli/metadata-backup.test.ts

@@ -38,7 +38,7 @@ import { join } from "node:path";
 import { tmpdir } from "node:os";
 import sodium from "libsodium-wrappers-sumo";
 import { SRP, SrpServer } from "fast-srp-hap";
-import { beforeAll, afterAll, describe, expect, it } from "vitest";
+import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
 import {
     init,
     toBase64,
@@ -53,8 +53,16 @@ import {
     runMetadataBackup,
     type MetadataBackupOptions,
 } from "../../src/metadata-backup.js";
+import { backupMetadataCommand } from "../../src/cli-commands.js";
 import type { KeyAttributes } from "../../src/auth/types.js";
 
+// One file per ML data request, so the two files of the mock account are
+// fetched in two requests and one of them can fail on its own.
+vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({
+    ...(await importOriginal<typeof import("../../src/mldata-fetch.js")>()),
+    MLDATA_BATCH_SIZE: 1,
+}));
+
 const TEST_EMAIL = "metabackup@example.com";
 const TEST_PASSWORD = "metapass";
 const TEST_OPS = 2;
@@ -347,7 +355,8 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
     };
 };
 
-const buildMetaFetch = (m: MetaMockState) => {
+// `failMLDataFor`: answer 500 to every ML data request that asks for this file.
+const buildMetaFetch = (m: MetaMockState, failMLDataFor?: number) => {
     let srpServer: SrpServer;
     return (async (
         input: RequestInfo | URL,
@@ -403,6 +412,8 @@ const buildMetaFetch = (m: MetaMockState) => {
         }
         if (path === "/files/data/fetch") {
             const body = JSON.parse(init?.body as string);
+            if ((body.fileIDs as number[]).includes(failMLDataFor!))
+                return new Response("server error", { status: 500 });
             const data = (body.fileIDs as number[])
                 .filter((id: number) => m.encryptedMLData[id])
                 .map((id: number) => ({
@@ -636,3 +647,63 @@ describe("quak backup-metadata", () => {
         expect(failedMeta.imageMetadataError).toEqual(expect.any(String));
     });
 });
+
+describe("quak backup-metadata when an ML data request fails", () => {
+    // Run the CLI command against the mock and return its exit code, stderr
+    // and output directory.
+    const runCommand = async (failMLDataFor?: number) => {
+        const client = await Client.login({
+            email: TEST_EMAIL,
+            password: TEST_PASSWORD,
+            apiOptions: {
+                fetch: buildMetaFetch(mock, failMLDataFor),
+                retry: { sleep: async () => {} },
+            },
+        });
+        const outDir = mkdtempSync(join(testDir, "ml-fail-"));
+        let stderr = "";
+        const code = await backupMetadataCommand(
+            {
+                stdout: { write: () => true },
+                stderr: { write: (text: string) => (stderr += text) },
+                sessionDir: testDir,
+                cacheDir: mkdtempSync(join(testDir, "cache-")),
+                loadSession: () => client,
+            },
+            outDir,
+            {},
+        );
+        return { code, stderr, outDir };
+    };
+
+    it("writes every file, marks the failed batch's files, and exits 1", async () => {
+        const { code, stderr, outDir } = await runCommand(200);
+
+        expect(code).toBe(1);
+        expect(stderr).toContain("ML data request for 1 file(s) failed");
+
+        const ok = JSON.parse(
+            readFileSync(
+                join(outDir, "collections", "10-Vacation", "100.json"),
+                "utf-8",
+            ),
+        );
+        expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
+        expect(ok.mlDataError).toBeUndefined();
+
+        const failed = JSON.parse(
+            readFileSync(
+                join(outDir, "collections", "20-__Work", "200.json"),
+                "utf-8",
+            ),
+        );
+        expect(failed.metadata.title).toBe("diagram.png");
+        expect(failed.mlData).toBeUndefined();
+        expect(failed.mlDataError).toContain("500");
+    });
+
+    it("exits 0 when every ML data request succeeds", async () => {
+        const { code } = await runCommand();
+        expect(code).toBe(0);
+    });
+});

test/crypto/hash.test.ts

@@ -0,0 +1,33 @@
+import { beforeAll, describe, expect, it } from "vitest";
+import {
+    chunkHashFinal,
+    chunkHashInit,
+    chunkHashUpdate,
+    init,
+} from "../../src/crypto/index.js";
+
+beforeAll(async () => {
+    await init();
+});
+
+describe("content hash", () => {
+    // RFC 7693 Appendix A: BLAKE2b-512 of "abc".
+    const abc = Buffer.from(
+        "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1" +
+            "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923",
+        "hex",
+    ).toString("base64");
+
+    it("is unkeyed BLAKE2b-512 in standard base64", () => {
+        const state = chunkHashInit();
+        chunkHashUpdate(state, new TextEncoder().encode("abc"));
+        expect(chunkHashFinal(state)).toBe(abc);
+    });
+
+    it("gives the same hash when the input arrives in chunks", () => {
+        const state = chunkHashInit();
+        chunkHashUpdate(state, new TextEncoder().encode("a"));
+        chunkHashUpdate(state, new TextEncoder().encode("bc"));
+        expect(chunkHashFinal(state)).toBe(abc);
+    });
+});

test/download/download.test.ts

@@ -61,6 +61,7 @@ import { dirname, join } from "node:path";
 import { tmpdir } from "node:os";
 import { createHash } from "node:crypto";
 import sodium from "libsodium-wrappers-sumo";
+import { zipSync } from "fflate";
 import {
     beforeAll,
     beforeEach,
@@ -188,7 +189,31 @@ vi.mock("node:fs/promises", async (importOriginal) => {
     };
 });
 
+/**
+ * `chunkHashUpdate` is wrapped to record the length of every piece hashed, so
+ * a test can show that a live photo entry reaches the hash in pieces far
+ * smaller than the entry, rather than decompressed whole first.
+ */
+const hashHook = vi.hoisted(() => ({
+    lengths: [] as number[],
+}));
+
+vi.mock("../../src/crypto/index.js", async (importOriginal) => {
+    const actual =
+        await importOriginal<typeof import("../../src/crypto/index.js")>();
+    return {
+        ...actual,
+        chunkHashUpdate: (
+            ...args: Parameters<typeof actual.chunkHashUpdate>
+        ): void => {
+            hashHook.lengths.push(args[1].length);
+            actual.chunkHashUpdate(...args);
+        },
+    };
+});
+
 beforeEach(() => {
+    hashHook.lengths.length = 0;
     renameHook.calls.length = 0;
     renameHook.failWith = null;
     durabilityHook.events.length = 0;
@@ -1613,3 +1638,113 @@ describe.each(entryPoints)("$name progress", ({ name, download }) => {
         expectSameBytes(readFileSync(outPath), plaintext);
     });
 });
+
+describe("downloadFile content hash", () => {
+    // Node's own BLAKE2b-512 is the reference, so these tests do not depend
+    // on the code under test to compute what they expect.
+    const blake2b = (bytes: Uint8Array): string =>
+        createHash("blake2b512").update(bytes).digest("base64");
+
+    // Serve `plaintext` encrypted as file 999 with the given metadata. Four
+    // responses are scripted so a retried mismatch would show in `requests`.
+    const setup = (plaintext: Uint8Array, metadata: Partial<FileMetadata>) => {
+        const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
+        const { header, ciphertext } = encryptFileBody(plaintext, key);
+        const file = buildMockEnteFile(key, header, header);
+        file.metadata = { ...file.metadata, ...metadata };
+        const body = { kind: "body", bytes: ciphertext } as const;
+        const { fetch, requests } = scriptedCdnFetch(body, body, body, body);
+        const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } });
+        const dir = mkdtempSync(join(testDir, "hash-"));
+        const outPath = join(dir, "f.bin");
+        return {
+            run: () => downloadFile(api, file, outPath),
+            dir,
+            outPath,
+            requests,
+        };
+    };
+
+    const livePhotoZip = zipSync({
+        "image.heic": patternBytes(500, 81),
+        "video.mov": patternBytes(900, 82),
+    });
+    const livePhotoHash = `${blake2b(patternBytes(500, 81))}:${blake2b(patternBytes(900, 82))}`;
+
+    it("stores a file whose hash matches", async () => {
+        const plaintext = patternBytes(700, 80);
+        const t = setup(plaintext, { hash: blake2b(plaintext) });
+
+        await t.run();
+
+        expectSameBytes(readFileSync(t.outPath), plaintext);
+    });
+
+    it("rejects a mismatch, stores nothing, names the file and does not retry", async () => {
+        const t = setup(patternBytes(700, 80), {
+            hash: blake2b(patternBytes(700, 79)),
+        });
+
+        await expect(t.run()).rejects.toThrow(
+            /file 999: content hash .* does not match/,
+        );
+
+        expect(readdirSync(t.dir)).toEqual([]);
+        expect(t.requests()).toBe(1);
+    });
+
+    it("stores a file with no recorded hash unchecked", async () => {
+        const plaintext = patternBytes(700, 80);
+        const t = setup(plaintext, { hash: undefined });
+
+        await t.run();
+
+        expectSameBytes(readFileSync(t.outPath), plaintext);
+    });
+
+    it("stores a live photo whose image and video hashes match", async () => {
+        const t = setup(livePhotoZip, {
+            fileType: "livePhoto",
+            hash: livePhotoHash,
+        });
+
+        await t.run();
+
+        expectSameBytes(readFileSync(t.outPath), livePhotoZip);
+    });
+
+    it("hashes a large live photo entry as it decompresses, never whole", async () => {
+        // 64 MiB of zeros deflates to a few kilobytes, the shape of a ZIP
+        // that would exhaust memory if expanded whole.
+        const image = new Uint8Array(64 * 1024 * 1024);
+        const video = patternBytes(900, 83);
+        const zip = zipSync({ "image.heic": image, "video.mov": video });
+        const t = setup(zip, {
+            fileType: "livePhoto",
+            hash: `${blake2b(image)}:${blake2b(video)}`,
+        });
+
+        await t.run();
+
+        expectSameBytes(readFileSync(t.outPath), zip);
+        const hashed = hashHook.lengths.reduce((a, b) => a + b, 0);
+        expect(hashed).toBe(image.length + video.length);
+        expect(Math.max(...hashHook.lengths)).toBeLessThanOrEqual(
+            2 * STREAM_CHUNK_SIZE,
+        );
+    });
+
+    it("rejects a live photo whose hash does not match", async () => {
+        // The whole ZIP's hash is not the recorded one: each part is hashed.
+        const t = setup(livePhotoZip, {
+            fileType: "livePhoto",
+            hash: blake2b(livePhotoZip),
+        });
+
+        await expect(t.run()).rejects.toThrow(
+            /file 999: content hash .* does not match/,
+        );
+
+        expect(readdirSync(t.dir)).toEqual([]);
+    });
+});

test/model/decrypt.test.ts

@@ -351,6 +351,46 @@ describe("model.decryptFile", () => {
         }
     });
 
+    it("reads the recorded content hash, joining an older live photo's two parts", () => {
+        const masterKey = sodium.crypto_secretbox_keygen();
+        const { collectionKey } = buildRawCollection(masterKey);
+        const hashOf = (metadata: Record<string, unknown>) =>
+            decryptFile(
+                buildRawFile(collectionKey, {
+                    metadata: { title: "x", ...metadata },
+                }),
+                collectionKey,
+            ).metadata.hash;
+
+        expect(hashOf({ fileType: 0, hash: "H" })).toBe("H");
+        expect(
+            hashOf({ fileType: 2, hash: "H", imageHash: "I", videoHash: "V" }),
+        ).toBe("H");
+        expect(hashOf({ fileType: 2, imageHash: "I", videoHash: "V" })).toBe(
+            "I:V",
+        );
+        expect(hashOf({ fileType: 2, imageHash: "I" })).toBeUndefined();
+        expect(
+            hashOf({ fileType: 0, imageHash: "I", videoHash: "V" }),
+        ).toBeUndefined();
+        expect(hashOf({ fileType: 0 })).toBeUndefined();
+        expect(hashOf({ fileType: 0, hash: 42 })).toBeUndefined();
+        expect(
+            hashOf({ fileType: 2, imageHash: "I", videoHash: 7 }),
+        ).toBeUndefined();
+        // An empty string counts as absent, not as a hash to match.
+        expect(hashOf({ fileType: 0, hash: "" })).toBeUndefined();
+        expect(
+            hashOf({ fileType: 2, hash: "", imageHash: "I", videoHash: "V" }),
+        ).toBe("I:V");
+        expect(
+            hashOf({ fileType: 2, imageHash: "", videoHash: "V" }),
+        ).toBeUndefined();
+        expect(
+            hashOf({ fileType: 2, imageHash: "I", videoHash: "" }),
+        ).toBeUndefined();
+    });
+
     it("maps fileType numbers to FileType strings", () => {
         // Ente uses: 0=image, 1=video, 2=livePhoto
         const masterKey = sodium.crypto_secretbox_keygen();

yarn.lock

@@ -1069,6 +1069,11 @@ fastq@^1.6.0:
   dependencies:
     reusify "^1.0.4"
 
+fflate@0.8.3:
+  version "0.8.3"
+  resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.3.tgz#bc27d8eb30343d4d512abb03480202ce65d825fc"
+  integrity sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==
+
 file-entry-cache@^8.0.0:
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f"