diff --git a/README.md b/README.md index b98882b..b06642f 100644 --- a/README.md +++ b/README.md @@ -677,10 +677,13 @@ Under `cacheDirectory`: ``` A stored file appears only via an atomic temp-then-rename, so its presence means -it is complete. The design also calls for a content-hash comparison against -`FileMetadata.hash` on each fetched original; that check is deferred (issue -https://git.eeqj.de/sneak/quak/issues/68) because the exact hash construction -cannot yet be confirmed against the repo's fixtures. +it is complete. Every downloaded original (by `quak get`, the cache, or +`backup`) whose metadata records a content hash (`FileMetadata.hash`) is hashed +as it is written: unkeyed BLAKE2b with a 64-byte output, standard base64. For a +live photo, which is stored as a ZIP, the image and the video are hashed +separately and joined as `:`. A mismatch stores nothing +and fails the download with an error naming the file ID. An original with no +recorded hash, from a very old client, is stored unchecked. ### Key types by source file diff --git a/TODO.md b/TODO.md index 0622278..4f3a2c9 100644 --- a/TODO.md +++ b/TODO.md @@ -18,6 +18,14 @@ Tag v1.0.0. # Completed Steps +- 2026-09-23: Checked downloaded originals against their recorded content hash + (issue 68). `downloadFile`, which `quak get`, the content cache and backup all + use, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and + stores nothing on a mismatch, failing with an error naming the file ID. A live + photo ZIP is unpacked with `fflate` and its image and video hashed separately + as `:`. `decryptFile` reads older clients' `imageHash` + and `videoHash` fields for live photos. A file with no recorded hash is stored + unchecked. - 2026-09-23: Single-sourced the version string (issue 5). `package.json` is the only place it is written: `src/index.ts` imports it for `VERSION` and `bin/quak.ts` passes `VERSION` to commander. tsc copies `package.json` to diff --git a/package.json b/package.json index 0db546f..7376620 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "env-paths": "4.0.0", "exif-reader": "2.0.3", "fast-srp-hap": "2.0.4", + "fflate": "0.8.3", "jpeg-js": "0.4.4", "libsodium-wrappers-sumo": "0.8.4" } diff --git a/src/crypto/hash.ts b/src/crypto/hash.ts new file mode 100644 index 0000000..b94d94f --- /dev/null +++ b/src/crypto/hash.ts @@ -0,0 +1,22 @@ +import sodium, { type StateAddress } from "libsodium-wrappers-sumo"; +import { toBase64 } from "./encoding.js"; + +// The content hash an uploading client records in a file's metadata: unkeyed +// BLAKE2b with a 64-byte output over the original's bytes, fed in chunks, as +// standard base64 with padding. Named after the upstream client's functions. +// The output length is read at call time for the same reason as +// `streamTagFinal` in stream.ts: libsodium sets its constants only once ready. + +export const chunkHashInit = (): StateAddress => + sodium.crypto_generichash_init(null, sodium.crypto_generichash_BYTES_MAX); + +export const chunkHashUpdate = (state: StateAddress, chunk: Uint8Array): void => + sodium.crypto_generichash_update(state, chunk); + +export const chunkHashFinal = (state: StateAddress): string => + toBase64( + sodium.crypto_generichash_final( + state, + sodium.crypto_generichash_BYTES_MAX, + ), + ); diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 4e3e53b..d93af09 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -7,6 +7,7 @@ export { } from "./encoding.js"; export { deriveKEK, deriveLoginSubkey } from "./kdf.js"; export { decryptBox, decryptSealed } from "./box.js"; +export { chunkHashFinal, chunkHashInit, chunkHashUpdate } from "./hash.js"; export { decryptBlob, encryptBlob, diff --git a/src/download/index.ts b/src/download/index.ts index 8606cd2..d01c34d 100644 --- a/src/download/index.ts +++ b/src/download/index.ts @@ -1,8 +1,12 @@ import { randomUUID } from "node:crypto"; -import { open, rename, rm } from "node:fs/promises"; +import { open, readFile, rename, rm } from "node:fs/promises"; import type { FileHandle } from "node:fs/promises"; import { dirname, join } from "node:path"; +import { unzipSync } from "fflate"; import { + chunkHashFinal, + chunkHashInit, + chunkHashUpdate, fromBase64, initStreamPull, pullStreamChunk, @@ -177,7 +181,8 @@ export const fsyncPath = async (path: string): Promise => { }; // Stage a write to `destination` atomically and durably, then rename it into -// place. `fill` writes the contents into the open temp file handle — either the +// place. `fill` writes the contents into the open temp file handle (whose path +// it is also given, so it can read back what it wrote) — either the // whole buffer at once (`writeAtomic`) or chunk by chunk as they decrypt // (`decryptToTemp`). The temp file is a sibling of the destination (same // directory, so the rename cannot cross a filesystem boundary), so callers @@ -197,7 +202,7 @@ export const fsyncPath = async (path: string): Promise => { // scratch file is left to fill the disk on repeated failures. const stageAtomic = async ( destination: string, - fill: (handle: FileHandle) => Promise, + fill: (handle: FileHandle, tmpPath: string) => Promise, ): Promise => { const dir = dirname(destination); // The random suffix keeps concurrent downloads of the same destination @@ -206,7 +211,7 @@ const stageAtomic = async ( try { const handle = await open(tmpPath, "w"); try { - await fill(handle); + await fill(handle, tmpPath); await handle.sync(); } finally { await handle.close(); @@ -237,31 +242,83 @@ export const writeAtomic = async ( ): Promise => stageAtomic(destination, (handle) => handle.writeFile(plaintext)); +const hashBytes = (bytes: Uint8Array): string => { + const state = chunkHashInit(); + chunkHashUpdate(state, bytes); + return chunkHashFinal(state); +}; + +// A live photo is stored as a ZIP of its image and its video, and its recorded +// hash is `:`, each over that part's own bytes. Like the +// upstream client's decoder, this takes the entries whose names start with +// `image` and `video`. The ZIP is read whole: a live photo is a few megabytes. +const livePhotoHash = async ( + fileID: number, + zipPath: string, +): Promise => { + let entries: [string, Uint8Array][]; + try { + entries = Object.entries(unzipSync(await readFile(zipPath))); + } catch (err) { + throw new Error(`download: file ${fileID}: live photo is not a ZIP`, { + cause: err, + }); + } + const image = entries.find(([name]) => name.startsWith("image")); + const video = entries.find(([name]) => name.startsWith("video")); + if (image === undefined || video === undefined) { + throw new Error( + `download: file ${fileID}: live photo ZIP does not hold both an image and a video`, + ); + } + return `${hashBytes(image[1])}:${hashBytes(video[1])}`; +}; + // Decrypt `stream` straight to `destination`, one plaintext chunk at a time, // under the atomic writer's temp-then-rename discipline. Memory stays bounded // by the chunk size: each decrypted chunk is written to the temp file and // dropped. The rename happens only after the stream authenticates as terminated // on TAG_FINAL; a truncated stream throws and leaves the destination untouched. // Returns the plaintext length written. +// +// `original` is the file whose original this is (none for a thumbnail, which +// has no recorded hash). When its metadata has a hash, the decrypted bytes +// must match it or nothing is stored: a plain file is hashed as it streams, a +// live photo from the finished ZIP. The mismatch error is not retried. const decryptToTemp = async ( destination: string, stream: ReadableStream, header: Uint8Array, key: Uint8Array, onProgress?: ProgressCallback, + original?: EnteFile, ): Promise => { + const expected = original?.metadata.hash; + const livePhoto = original?.metadata.fileType === "livePhoto"; + const hashState = + expected !== undefined && !livePhoto ? chunkHashInit() : undefined; let bytesWritten = 0; try { - await stageAtomic(destination, async (handle) => { + await stageAtomic(destination, async (handle, tmpPath) => { bytesWritten = await streamDecrypt( stream, header, key, async (plaintext) => { + if (hashState) chunkHashUpdate(hashState, plaintext); await handle.write(plaintext); }, onProgress, ); + if (original === undefined || expected === undefined) return; + const actual = hashState + ? chunkHashFinal(hashState) + : await livePhotoHash(original.id, tmpPath); + if (actual !== expected) { + throw new Error( + `download: file ${original.id}: content hash ${actual} does not match the hash its uploader recorded, ${expected}`, + ); + } }); } catch (err) { // Cancel the body so its connection is closed now rather than held @@ -302,10 +359,18 @@ const fetchAndDecrypt = async ( key: Uint8Array, destination: string, onProgress?: ProgressCallback, + original?: EnteFile, ): Promise => withRetry(async () => { const stream = await openStream(); - return decryptToTemp(destination, stream, header, key, onProgress); + return decryptToTemp( + destination, + stream, + header, + key, + onProgress, + original, + ); }, api.getRetryOptions()); export const downloadFile = async ( @@ -326,6 +391,7 @@ export const downloadFile = async ( file.key, resolvedPath, onProgress, + file, ); return { path: resolvedPath, bytesWritten }; }; diff --git a/src/library/content.ts b/src/library/content.ts index c5c4c6e..eb52948 100644 --- a/src/library/content.ts +++ b/src/library/content.ts @@ -15,12 +15,12 @@ // Integrity. The reused streaming decrypt is the enforced guarantee: every // chunk is authenticated and the writer renames the file into place only once // the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and -// nothing is stored. On top of that this module refuses to record a stored file -// that came out empty. The design also asks for a content-hash comparison -// against `FileMetadata.hash` (with a `fileSize` fallback); that is deferred — -// see the PR — because the exact hash construction cannot be confirmed against -// the repo's fixtures and `FileBlob.size` is the encrypted object size, not the -// decrypted length this layer has. +// nothing is stored. For an original whose metadata records a content hash +// (`FileMetadata.hash`), the writer also hashes the decrypted bytes and stores +// nothing if they differ, failing the fetch with an error naming the file. An +// original with no recorded hash is stored unchecked, as the upstream client +// does; thumbnails have none. On top of that this module refuses to record a +// stored file that came out empty. import { existsSync, statSync } from "node:fs"; import { diff --git a/src/model/decrypt.ts b/src/model/decrypt.ts index 4458b4c..9137479 100644 --- a/src/model/decrypt.ts +++ b/src/model/decrypt.ts @@ -34,6 +34,28 @@ const FILE_TYPE_MAP: Record = { const parseFileType = (n: number): FileType => FILE_TYPE_MAP[n] ?? "unknown"; +// The hash the uploading client recorded for the original's bytes, read the +// way the upstream client's `metadataHash` reads it: `hash` if present, +// otherwise, for a live photo from an older client that wrote the two parts +// separately, `:`. A field that is not a non-empty +// string counts as absent, and a file with no hash at all is normal. +const expectedHash = (json: Record): string | undefined => { + const text = (v: unknown): string | undefined => + typeof v === "string" && v !== "" ? v : undefined; + const hash = text(json.hash); + if (hash !== undefined) return hash; + const imageHash = text(json.imageHash); + const videoHash = text(json.videoHash); + if ( + json.fileType === 2 && + imageHash !== undefined && + videoHash !== undefined + ) { + return `${imageHash}:${videoHash}`; + } + return undefined; +}; + export const decryptCollection = ( raw: RawCollection, keys: KeyMaterial, @@ -115,7 +137,7 @@ export const decryptFile = ( modificationTime: metadataJSON.modificationTime ?? 0, latitude: metadataJSON.latitude, longitude: metadataJSON.longitude, - hash: metadataJSON.hash, + hash: expectedHash(metadataJSON), }; const magicMetadata = decryptMagicMetadata(raw.magicMetadata, key); diff --git a/src/model/types.ts b/src/model/types.ts index 65cc845..516407a 100644 --- a/src/model/types.ts +++ b/src/model/types.ts @@ -29,6 +29,9 @@ export interface FileMetadata { modificationTime: Microseconds; latitude?: number; longitude?: number; + // The content hash the uploader recorded (see `expectedHash` in + // decrypt.ts); `downloadFile` refuses an original that does not match it. + // Absent for files from very old clients. hash?: string; } diff --git a/test/crypto/hash.test.ts b/test/crypto/hash.test.ts new file mode 100644 index 0000000..153ec0a --- /dev/null +++ b/test/crypto/hash.test.ts @@ -0,0 +1,33 @@ +import { beforeAll, describe, expect, it } from "vitest"; +import { + chunkHashFinal, + chunkHashInit, + chunkHashUpdate, + init, +} from "../../src/crypto/index.js"; + +beforeAll(async () => { + await init(); +}); + +describe("content hash", () => { + // RFC 7693 Appendix A: BLAKE2b-512 of "abc". + const abc = Buffer.from( + "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1" + + "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923", + "hex", + ).toString("base64"); + + it("is unkeyed BLAKE2b-512 in standard base64", () => { + const state = chunkHashInit(); + chunkHashUpdate(state, new TextEncoder().encode("abc")); + expect(chunkHashFinal(state)).toBe(abc); + }); + + it("gives the same hash when the input arrives in chunks", () => { + const state = chunkHashInit(); + chunkHashUpdate(state, new TextEncoder().encode("a")); + chunkHashUpdate(state, new TextEncoder().encode("bc")); + expect(chunkHashFinal(state)).toBe(abc); + }); +}); diff --git a/test/download/download.test.ts b/test/download/download.test.ts index 7b70760..c65b6fd 100644 --- a/test/download/download.test.ts +++ b/test/download/download.test.ts @@ -61,6 +61,7 @@ import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; import { createHash } from "node:crypto"; import sodium from "libsodium-wrappers-sumo"; +import { zipSync } from "fflate"; import { beforeAll, beforeEach, @@ -1613,3 +1614,92 @@ describe.each(entryPoints)("$name progress", ({ name, download }) => { expectSameBytes(readFileSync(outPath), plaintext); }); }); + +describe("downloadFile content hash", () => { + // Node's own BLAKE2b-512 is the reference, so these tests do not depend + // on the code under test to compute what they expect. + const blake2b = (bytes: Uint8Array): string => + createHash("blake2b512").update(bytes).digest("base64"); + + // Serve `plaintext` encrypted as file 999 with the given metadata. Four + // responses are scripted so a retried mismatch would show in `requests`. + const setup = (plaintext: Uint8Array, metadata: Partial) => { + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptFileBody(plaintext, key); + const file = buildMockEnteFile(key, header, header); + file.metadata = { ...file.metadata, ...metadata }; + const body = { kind: "body", bytes: ciphertext } as const; + const { fetch, requests } = scriptedCdnFetch(body, body, body, body); + const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } }); + const dir = mkdtempSync(join(testDir, "hash-")); + const outPath = join(dir, "f.bin"); + return { + run: () => downloadFile(api, file, outPath), + dir, + outPath, + requests, + }; + }; + + const livePhotoZip = zipSync({ + "image.heic": patternBytes(500, 81), + "video.mov": patternBytes(900, 82), + }); + const livePhotoHash = `${blake2b(patternBytes(500, 81))}:${blake2b(patternBytes(900, 82))}`; + + it("stores a file whose hash matches", async () => { + const plaintext = patternBytes(700, 80); + const t = setup(plaintext, { hash: blake2b(plaintext) }); + + await t.run(); + + expectSameBytes(readFileSync(t.outPath), plaintext); + }); + + it("rejects a mismatch, stores nothing, names the file and does not retry", async () => { + const t = setup(patternBytes(700, 80), { + hash: blake2b(patternBytes(700, 79)), + }); + + await expect(t.run()).rejects.toThrow( + /file 999: content hash .* does not match/, + ); + + expect(readdirSync(t.dir)).toEqual([]); + expect(t.requests()).toBe(1); + }); + + it("stores a file with no recorded hash unchecked", async () => { + const plaintext = patternBytes(700, 80); + const t = setup(plaintext, { hash: undefined }); + + await t.run(); + + expectSameBytes(readFileSync(t.outPath), plaintext); + }); + + it("stores a live photo whose image and video hashes match", async () => { + const t = setup(livePhotoZip, { + fileType: "livePhoto", + hash: livePhotoHash, + }); + + await t.run(); + + expectSameBytes(readFileSync(t.outPath), livePhotoZip); + }); + + it("rejects a live photo whose hash does not match", async () => { + // The whole ZIP's hash is not the recorded one: each part is hashed. + const t = setup(livePhotoZip, { + fileType: "livePhoto", + hash: blake2b(livePhotoZip), + }); + + await expect(t.run()).rejects.toThrow( + /file 999: content hash .* does not match/, + ); + + expect(readdirSync(t.dir)).toEqual([]); + }); +}); diff --git a/test/model/decrypt.test.ts b/test/model/decrypt.test.ts index 76dac9e..7d0c2c0 100644 --- a/test/model/decrypt.test.ts +++ b/test/model/decrypt.test.ts @@ -351,6 +351,35 @@ describe("model.decryptFile", () => { } }); + it("reads the recorded content hash, joining an older live photo's two parts", () => { + const masterKey = sodium.crypto_secretbox_keygen(); + const { collectionKey } = buildRawCollection(masterKey); + const hashOf = (metadata: Record) => + decryptFile( + buildRawFile(collectionKey, { + metadata: { title: "x", ...metadata }, + }), + collectionKey, + ).metadata.hash; + + expect(hashOf({ fileType: 0, hash: "H" })).toBe("H"); + expect( + hashOf({ fileType: 2, hash: "H", imageHash: "I", videoHash: "V" }), + ).toBe("H"); + expect(hashOf({ fileType: 2, imageHash: "I", videoHash: "V" })).toBe( + "I:V", + ); + expect(hashOf({ fileType: 2, imageHash: "I" })).toBeUndefined(); + expect( + hashOf({ fileType: 0, imageHash: "I", videoHash: "V" }), + ).toBeUndefined(); + expect(hashOf({ fileType: 0 })).toBeUndefined(); + expect(hashOf({ fileType: 0, hash: 42 })).toBeUndefined(); + expect( + hashOf({ fileType: 2, imageHash: "I", videoHash: 7 }), + ).toBeUndefined(); + }); + it("maps fileType numbers to FileType strings", () => { // Ente uses: 0=image, 1=video, 2=livePhoto const masterKey = sodium.crypto_secretbox_keygen(); diff --git a/yarn.lock b/yarn.lock index d3a5f20..7759c2d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1069,6 +1069,11 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" +fflate@0.8.3: + version "0.8.3" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.3.tgz#bc27d8eb30343d4d512abb03480202ce65d825fc" + integrity sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA== + file-entry-cache@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f"