Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e238bc1bd | ||
|
|
c19943a520 |
@@ -698,10 +698,13 @@ originals and thumbnails are kept; they are reached only through the files the
|
|||||||
current account's records name.
|
current account's records name.
|
||||||
|
|
||||||
A stored file appears only via an atomic temp-then-rename, so its presence means
|
A stored file appears only via an atomic temp-then-rename, so its presence means
|
||||||
it is complete. The design also calls for a content-hash comparison against
|
it is complete. Every downloaded original (by `quak get`, the cache, or
|
||||||
`FileMetadata.hash` on each fetched original; that check is deferred (issue
|
`backup`) whose metadata records a content hash (`FileMetadata.hash`) is hashed
|
||||||
https://git.eeqj.de/sneak/quak/issues/68) because the exact hash construction
|
as it is written: unkeyed BLAKE2b with a 64-byte output, standard base64. For a
|
||||||
cannot yet be confirmed against the repo's fixtures.
|
live photo, which is stored as a ZIP, the image and the video are hashed
|
||||||
|
separately and joined as `<imageHash>:<videoHash>`. A mismatch stores nothing
|
||||||
|
and fails the download with an error naming the file ID. An original with no
|
||||||
|
recorded hash, from a very old client, is stored unchecked.
|
||||||
|
|
||||||
### Key types by source file
|
### Key types by source file
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,15 @@ Tag v1.0.0.
|
|||||||
cache path is now `defaultCacheDirectory()` in the library, shared with
|
cache path is now `defaultCacheDirectory()` in the library, shared with
|
||||||
`Library.open`.
|
`Library.open`.
|
||||||
|
|
||||||
|
- 2026-09-23: Checked downloaded originals against their recorded content hash
|
||||||
|
(issue 68). `downloadFile`, which `quak get`, the content cache and backup all
|
||||||
|
use, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and
|
||||||
|
stores nothing on a mismatch, failing with an error naming the file ID. A live
|
||||||
|
photo ZIP is unpacked as it streams with `fflate` and its image and video
|
||||||
|
hashed separately as `<imageHash>:<videoHash>`. `decryptFile` reads older
|
||||||
|
clients' `imageHash` and `videoHash` fields for live photos. A file with no
|
||||||
|
recorded hash is stored unchecked.
|
||||||
|
|
||||||
- 2026-09-23: Kept one account's cache from mixing with another's (issue 104).
|
- 2026-09-23: Kept one account's cache from mixing with another's (issue 104).
|
||||||
When `metadata.json` in the cache directory was written for a different,
|
When `metadata.json` in the cache directory was written for a different,
|
||||||
non-zero user ID than the client's, `Library.open` deletes it and `mldata/`
|
non-zero user ID than the client's, `Library.open` deletes it and `mldata/`
|
||||||
|
|||||||
@@ -42,6 +42,7 @@
|
|||||||
"env-paths": "4.0.0",
|
"env-paths": "4.0.0",
|
||||||
"exif-reader": "2.0.3",
|
"exif-reader": "2.0.3",
|
||||||
"fast-srp-hap": "2.0.4",
|
"fast-srp-hap": "2.0.4",
|
||||||
|
"fflate": "0.8.3",
|
||||||
"jpeg-js": "0.4.4",
|
"jpeg-js": "0.4.4",
|
||||||
"libsodium-wrappers-sumo": "0.8.4"
|
"libsodium-wrappers-sumo": "0.8.4"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
import sodium, { type StateAddress } from "libsodium-wrappers-sumo";
|
||||||
|
import { toBase64 } from "./encoding.js";
|
||||||
|
|
||||||
|
// The content hash an uploading client records in a file's metadata: unkeyed
|
||||||
|
// BLAKE2b with a 64-byte output over the original's bytes, fed in chunks, as
|
||||||
|
// standard base64 with padding. Named after the upstream client's functions.
|
||||||
|
// The output length is read at call time for the same reason as
|
||||||
|
// `streamTagFinal` in stream.ts: libsodium sets its constants only once ready.
|
||||||
|
|
||||||
|
export const chunkHashInit = (): StateAddress =>
|
||||||
|
sodium.crypto_generichash_init(null, sodium.crypto_generichash_BYTES_MAX);
|
||||||
|
|
||||||
|
export const chunkHashUpdate = (state: StateAddress, chunk: Uint8Array): void =>
|
||||||
|
sodium.crypto_generichash_update(state, chunk);
|
||||||
|
|
||||||
|
export const chunkHashFinal = (state: StateAddress): string =>
|
||||||
|
toBase64(
|
||||||
|
sodium.crypto_generichash_final(
|
||||||
|
state,
|
||||||
|
sodium.crypto_generichash_BYTES_MAX,
|
||||||
|
),
|
||||||
|
);
|
||||||
@@ -7,6 +7,7 @@ export {
|
|||||||
} from "./encoding.js";
|
} from "./encoding.js";
|
||||||
export { deriveKEK, deriveLoginSubkey } from "./kdf.js";
|
export { deriveKEK, deriveLoginSubkey } from "./kdf.js";
|
||||||
export { decryptBox, decryptSealed } from "./box.js";
|
export { decryptBox, decryptSealed } from "./box.js";
|
||||||
|
export { chunkHashFinal, chunkHashInit, chunkHashUpdate } from "./hash.js";
|
||||||
export {
|
export {
|
||||||
decryptBlob,
|
decryptBlob,
|
||||||
encryptBlob,
|
encryptBlob,
|
||||||
|
|||||||
+112
-1
@@ -2,7 +2,11 @@ import { randomUUID } from "node:crypto";
|
|||||||
import { open, rename, rm } from "node:fs/promises";
|
import { open, rename, rm } from "node:fs/promises";
|
||||||
import type { FileHandle } from "node:fs/promises";
|
import type { FileHandle } from "node:fs/promises";
|
||||||
import { dirname, join } from "node:path";
|
import { dirname, join } from "node:path";
|
||||||
|
import { Unzip, UnzipInflate } from "fflate";
|
||||||
import {
|
import {
|
||||||
|
chunkHashFinal,
|
||||||
|
chunkHashInit,
|
||||||
|
chunkHashUpdate,
|
||||||
fromBase64,
|
fromBase64,
|
||||||
initStreamPull,
|
initStreamPull,
|
||||||
pullStreamChunk,
|
pullStreamChunk,
|
||||||
@@ -237,19 +241,109 @@ export const writeAtomic = async (
|
|||||||
): Promise<void> =>
|
): Promise<void> =>
|
||||||
stageAtomic(destination, (handle) => handle.writeFile(plaintext));
|
stageAtomic(destination, (handle) => handle.writeFile(plaintext));
|
||||||
|
|
||||||
|
// Hashes an original's bytes as they are decrypted, for comparison with the
|
||||||
|
// hash its uploader recorded.
|
||||||
|
interface ContentHasher {
|
||||||
|
update: (plaintext: Uint8Array) => void;
|
||||||
|
digest: () => string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileHasher = (): ContentHasher => {
|
||||||
|
const state = chunkHashInit();
|
||||||
|
return {
|
||||||
|
update: (plaintext) => chunkHashUpdate(state, plaintext),
|
||||||
|
digest: () => chunkHashFinal(state),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// A live photo is stored as a ZIP of its image and its video, and its recorded
|
||||||
|
// hash is `<imageHash>:<videoHash>`, each over that part's own bytes. Like the
|
||||||
|
// upstream client's decoder, this takes the first entries whose names start
|
||||||
|
// with `image` and `video`.
|
||||||
|
//
|
||||||
|
// The ZIP is chosen by its uploader and may expand enormously, so entries are
|
||||||
|
// hashed as they decompress and never held. fflate's `Unzip` inflates each
|
||||||
|
// push in one piece, and deflate expands at most about 1000-fold, so the ZIP
|
||||||
|
// is pushed in 4 KiB slices to keep each decompressed piece near 4 MiB, one
|
||||||
|
// plaintext chunk. Every entry is started, even one that is not hashed,
|
||||||
|
// because fflate keeps an unstarted entry's data in memory.
|
||||||
|
const livePhotoHasher = (fileID: number): ContentHasher => {
|
||||||
|
const sliceSize = 4096;
|
||||||
|
const fail = (message: string, cause?: unknown): Error =>
|
||||||
|
new Error(`download: file ${fileID}: ${message}`, { cause });
|
||||||
|
const claimed = new Set<string>();
|
||||||
|
const hashes = new Map<string, string>();
|
||||||
|
const unzip = new Unzip((entry) => {
|
||||||
|
const part = ["image", "video"].find((p) => entry.name.startsWith(p));
|
||||||
|
const target =
|
||||||
|
part === undefined || claimed.has(part)
|
||||||
|
? undefined
|
||||||
|
: { part, state: chunkHashInit() };
|
||||||
|
if (target !== undefined) claimed.add(target.part);
|
||||||
|
entry.ondata = (err, data, final) => {
|
||||||
|
if (err) throw err;
|
||||||
|
if (target === undefined) return;
|
||||||
|
chunkHashUpdate(target.state, data);
|
||||||
|
if (final) hashes.set(target.part, chunkHashFinal(target.state));
|
||||||
|
};
|
||||||
|
entry.start();
|
||||||
|
});
|
||||||
|
unzip.register(UnzipInflate);
|
||||||
|
// fflate reports a bad ZIP by throwing, sometimes a TypeError, which the
|
||||||
|
// retry would take for a network failure; a bad ZIP is never retried.
|
||||||
|
const push = (data: Uint8Array, final: boolean): void => {
|
||||||
|
try {
|
||||||
|
unzip.push(data, final);
|
||||||
|
} catch (err) {
|
||||||
|
throw fail("live photo is not a readable ZIP", err);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
update: (plaintext) => {
|
||||||
|
for (let i = 0; i < plaintext.length; i += sliceSize) {
|
||||||
|
push(plaintext.subarray(i, i + sliceSize), false);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
digest: () => {
|
||||||
|
push(new Uint8Array(0), true);
|
||||||
|
const image = hashes.get("image");
|
||||||
|
const video = hashes.get("video");
|
||||||
|
if (image === undefined || video === undefined) {
|
||||||
|
throw fail(
|
||||||
|
"live photo ZIP does not hold both an image and a video",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return `${image}:${video}`;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
// Decrypt `stream` straight to `destination`, one plaintext chunk at a time,
|
// Decrypt `stream` straight to `destination`, one plaintext chunk at a time,
|
||||||
// under the atomic writer's temp-then-rename discipline. Memory stays bounded
|
// under the atomic writer's temp-then-rename discipline. Memory stays bounded
|
||||||
// by the chunk size: each decrypted chunk is written to the temp file and
|
// by the chunk size: each decrypted chunk is written to the temp file and
|
||||||
// dropped. The rename happens only after the stream authenticates as terminated
|
// dropped. The rename happens only after the stream authenticates as terminated
|
||||||
// on TAG_FINAL; a truncated stream throws and leaves the destination untouched.
|
// on TAG_FINAL; a truncated stream throws and leaves the destination untouched.
|
||||||
// Returns the plaintext length written.
|
// Returns the plaintext length written.
|
||||||
|
//
|
||||||
|
// `original` is the file whose original this is (none for a thumbnail, which
|
||||||
|
// has no recorded hash). When its metadata has a hash, the decrypted bytes
|
||||||
|
// must match it or nothing is stored. Both a plain file and a live photo's
|
||||||
|
// parts are hashed as they stream. The mismatch error is not retried.
|
||||||
const decryptToTemp = async (
|
const decryptToTemp = async (
|
||||||
destination: string,
|
destination: string,
|
||||||
stream: ReadableStream<Uint8Array>,
|
stream: ReadableStream<Uint8Array>,
|
||||||
header: Uint8Array,
|
header: Uint8Array,
|
||||||
key: Uint8Array,
|
key: Uint8Array,
|
||||||
onProgress?: ProgressCallback,
|
onProgress?: ProgressCallback,
|
||||||
|
original?: EnteFile,
|
||||||
): Promise<number> => {
|
): Promise<number> => {
|
||||||
|
const expected = original?.metadata.hash;
|
||||||
|
const hasher =
|
||||||
|
original === undefined || expected === undefined
|
||||||
|
? undefined
|
||||||
|
: original.metadata.fileType === "livePhoto"
|
||||||
|
? livePhotoHasher(original.id)
|
||||||
|
: fileHasher();
|
||||||
let bytesWritten = 0;
|
let bytesWritten = 0;
|
||||||
try {
|
try {
|
||||||
await stageAtomic(destination, async (handle) => {
|
await stageAtomic(destination, async (handle) => {
|
||||||
@@ -258,10 +352,18 @@ const decryptToTemp = async (
|
|||||||
header,
|
header,
|
||||||
key,
|
key,
|
||||||
async (plaintext) => {
|
async (plaintext) => {
|
||||||
|
hasher?.update(plaintext);
|
||||||
await handle.write(plaintext);
|
await handle.write(plaintext);
|
||||||
},
|
},
|
||||||
onProgress,
|
onProgress,
|
||||||
);
|
);
|
||||||
|
if (original === undefined || hasher === undefined) return;
|
||||||
|
const actual = hasher.digest();
|
||||||
|
if (actual !== expected) {
|
||||||
|
throw new Error(
|
||||||
|
`download: file ${original.id}: content hash ${actual} does not match the hash its uploader recorded, ${expected}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// Cancel the body so its connection is closed now rather than held
|
// Cancel the body so its connection is closed now rather than held
|
||||||
@@ -302,10 +404,18 @@ const fetchAndDecrypt = async (
|
|||||||
key: Uint8Array,
|
key: Uint8Array,
|
||||||
destination: string,
|
destination: string,
|
||||||
onProgress?: ProgressCallback,
|
onProgress?: ProgressCallback,
|
||||||
|
original?: EnteFile,
|
||||||
): Promise<number> =>
|
): Promise<number> =>
|
||||||
withRetry(async () => {
|
withRetry(async () => {
|
||||||
const stream = await openStream();
|
const stream = await openStream();
|
||||||
return decryptToTemp(destination, stream, header, key, onProgress);
|
return decryptToTemp(
|
||||||
|
destination,
|
||||||
|
stream,
|
||||||
|
header,
|
||||||
|
key,
|
||||||
|
onProgress,
|
||||||
|
original,
|
||||||
|
);
|
||||||
}, api.getRetryOptions());
|
}, api.getRetryOptions());
|
||||||
|
|
||||||
export const downloadFile = async (
|
export const downloadFile = async (
|
||||||
@@ -326,6 +436,7 @@ export const downloadFile = async (
|
|||||||
file.key,
|
file.key,
|
||||||
resolvedPath,
|
resolvedPath,
|
||||||
onProgress,
|
onProgress,
|
||||||
|
file,
|
||||||
);
|
);
|
||||||
return { path: resolvedPath, bytesWritten };
|
return { path: resolvedPath, bytesWritten };
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -15,12 +15,12 @@
|
|||||||
// Integrity. The reused streaming decrypt is the enforced guarantee: every
|
// Integrity. The reused streaming decrypt is the enforced guarantee: every
|
||||||
// chunk is authenticated and the writer renames the file into place only once
|
// chunk is authenticated and the writer renames the file into place only once
|
||||||
// the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and
|
// the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and
|
||||||
// nothing is stored. On top of that this module refuses to record a stored file
|
// nothing is stored. For an original whose metadata records a content hash
|
||||||
// that came out empty. The design also asks for a content-hash comparison
|
// (`FileMetadata.hash`), the writer also hashes the decrypted bytes and stores
|
||||||
// against `FileMetadata.hash` (with a `fileSize` fallback); that is deferred —
|
// nothing if they differ, failing the fetch with an error naming the file. An
|
||||||
// see the PR — because the exact hash construction cannot be confirmed against
|
// original with no recorded hash is stored unchecked, as the upstream client
|
||||||
// the repo's fixtures and `FileBlob.size` is the encrypted object size, not the
|
// does; thumbnails have none. On top of that this module refuses to record a
|
||||||
// decrypted length this layer has.
|
// stored file that came out empty.
|
||||||
|
|
||||||
import { existsSync, statSync } from "node:fs";
|
import { existsSync, statSync } from "node:fs";
|
||||||
import {
|
import {
|
||||||
|
|||||||
+23
-1
@@ -34,6 +34,28 @@ const FILE_TYPE_MAP: Record<number, FileType> = {
|
|||||||
|
|
||||||
const parseFileType = (n: number): FileType => FILE_TYPE_MAP[n] ?? "unknown";
|
const parseFileType = (n: number): FileType => FILE_TYPE_MAP[n] ?? "unknown";
|
||||||
|
|
||||||
|
// The hash the uploading client recorded for the original's bytes, read the
|
||||||
|
// way the upstream client's `metadataHash` reads it: `hash` if present,
|
||||||
|
// otherwise, for a live photo from an older client that wrote the two parts
|
||||||
|
// separately, `<imageHash>:<videoHash>`. A field that is not a non-empty
|
||||||
|
// string counts as absent, and a file with no hash at all is normal.
|
||||||
|
const expectedHash = (json: Record<string, unknown>): string | undefined => {
|
||||||
|
const text = (v: unknown): string | undefined =>
|
||||||
|
typeof v === "string" && v !== "" ? v : undefined;
|
||||||
|
const hash = text(json.hash);
|
||||||
|
if (hash !== undefined) return hash;
|
||||||
|
const imageHash = text(json.imageHash);
|
||||||
|
const videoHash = text(json.videoHash);
|
||||||
|
if (
|
||||||
|
json.fileType === 2 &&
|
||||||
|
imageHash !== undefined &&
|
||||||
|
videoHash !== undefined
|
||||||
|
) {
|
||||||
|
return `${imageHash}:${videoHash}`;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
export const decryptCollection = (
|
export const decryptCollection = (
|
||||||
raw: RawCollection,
|
raw: RawCollection,
|
||||||
keys: KeyMaterial,
|
keys: KeyMaterial,
|
||||||
@@ -115,7 +137,7 @@ export const decryptFile = (
|
|||||||
modificationTime: metadataJSON.modificationTime ?? 0,
|
modificationTime: metadataJSON.modificationTime ?? 0,
|
||||||
latitude: metadataJSON.latitude,
|
latitude: metadataJSON.latitude,
|
||||||
longitude: metadataJSON.longitude,
|
longitude: metadataJSON.longitude,
|
||||||
hash: metadataJSON.hash,
|
hash: expectedHash(metadataJSON),
|
||||||
};
|
};
|
||||||
|
|
||||||
const magicMetadata = decryptMagicMetadata(raw.magicMetadata, key);
|
const magicMetadata = decryptMagicMetadata(raw.magicMetadata, key);
|
||||||
|
|||||||
@@ -29,6 +29,9 @@ export interface FileMetadata {
|
|||||||
modificationTime: Microseconds;
|
modificationTime: Microseconds;
|
||||||
latitude?: number;
|
latitude?: number;
|
||||||
longitude?: number;
|
longitude?: number;
|
||||||
|
// The content hash the uploader recorded (see `expectedHash` in
|
||||||
|
// decrypt.ts); `downloadFile` refuses an original that does not match it.
|
||||||
|
// Absent for files from very old clients.
|
||||||
hash?: string;
|
hash?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
chunkHashFinal,
|
||||||
|
chunkHashInit,
|
||||||
|
chunkHashUpdate,
|
||||||
|
init,
|
||||||
|
} from "../../src/crypto/index.js";
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
await init();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("content hash", () => {
|
||||||
|
// RFC 7693 Appendix A: BLAKE2b-512 of "abc".
|
||||||
|
const abc = Buffer.from(
|
||||||
|
"ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1" +
|
||||||
|
"7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923",
|
||||||
|
"hex",
|
||||||
|
).toString("base64");
|
||||||
|
|
||||||
|
it("is unkeyed BLAKE2b-512 in standard base64", () => {
|
||||||
|
const state = chunkHashInit();
|
||||||
|
chunkHashUpdate(state, new TextEncoder().encode("abc"));
|
||||||
|
expect(chunkHashFinal(state)).toBe(abc);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("gives the same hash when the input arrives in chunks", () => {
|
||||||
|
const state = chunkHashInit();
|
||||||
|
chunkHashUpdate(state, new TextEncoder().encode("a"));
|
||||||
|
chunkHashUpdate(state, new TextEncoder().encode("bc"));
|
||||||
|
expect(chunkHashFinal(state)).toBe(abc);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -61,6 +61,7 @@ import { dirname, join } from "node:path";
|
|||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { createHash } from "node:crypto";
|
import { createHash } from "node:crypto";
|
||||||
import sodium from "libsodium-wrappers-sumo";
|
import sodium from "libsodium-wrappers-sumo";
|
||||||
|
import { zipSync } from "fflate";
|
||||||
import {
|
import {
|
||||||
beforeAll,
|
beforeAll,
|
||||||
beforeEach,
|
beforeEach,
|
||||||
@@ -188,7 +189,31 @@ vi.mock("node:fs/promises", async (importOriginal) => {
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* `chunkHashUpdate` is wrapped to record the length of every piece hashed, so
|
||||||
|
* a test can show that a live photo entry reaches the hash in pieces far
|
||||||
|
* smaller than the entry, rather than decompressed whole first.
|
||||||
|
*/
|
||||||
|
const hashHook = vi.hoisted(() => ({
|
||||||
|
lengths: [] as number[],
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../../src/crypto/index.js", async (importOriginal) => {
|
||||||
|
const actual =
|
||||||
|
await importOriginal<typeof import("../../src/crypto/index.js")>();
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
chunkHashUpdate: (
|
||||||
|
...args: Parameters<typeof actual.chunkHashUpdate>
|
||||||
|
): void => {
|
||||||
|
hashHook.lengths.push(args[1].length);
|
||||||
|
actual.chunkHashUpdate(...args);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
|
hashHook.lengths.length = 0;
|
||||||
renameHook.calls.length = 0;
|
renameHook.calls.length = 0;
|
||||||
renameHook.failWith = null;
|
renameHook.failWith = null;
|
||||||
durabilityHook.events.length = 0;
|
durabilityHook.events.length = 0;
|
||||||
@@ -1613,3 +1638,113 @@ describe.each(entryPoints)("$name progress", ({ name, download }) => {
|
|||||||
expectSameBytes(readFileSync(outPath), plaintext);
|
expectSameBytes(readFileSync(outPath), plaintext);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("downloadFile content hash", () => {
|
||||||
|
// Node's own BLAKE2b-512 is the reference, so these tests do not depend
|
||||||
|
// on the code under test to compute what they expect.
|
||||||
|
const blake2b = (bytes: Uint8Array): string =>
|
||||||
|
createHash("blake2b512").update(bytes).digest("base64");
|
||||||
|
|
||||||
|
// Serve `plaintext` encrypted as file 999 with the given metadata. Four
|
||||||
|
// responses are scripted so a retried mismatch would show in `requests`.
|
||||||
|
const setup = (plaintext: Uint8Array, metadata: Partial<FileMetadata>) => {
|
||||||
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
||||||
|
const file = buildMockEnteFile(key, header, header);
|
||||||
|
file.metadata = { ...file.metadata, ...metadata };
|
||||||
|
const body = { kind: "body", bytes: ciphertext } as const;
|
||||||
|
const { fetch, requests } = scriptedCdnFetch(body, body, body, body);
|
||||||
|
const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } });
|
||||||
|
const dir = mkdtempSync(join(testDir, "hash-"));
|
||||||
|
const outPath = join(dir, "f.bin");
|
||||||
|
return {
|
||||||
|
run: () => downloadFile(api, file, outPath),
|
||||||
|
dir,
|
||||||
|
outPath,
|
||||||
|
requests,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const livePhotoZip = zipSync({
|
||||||
|
"image.heic": patternBytes(500, 81),
|
||||||
|
"video.mov": patternBytes(900, 82),
|
||||||
|
});
|
||||||
|
const livePhotoHash = `${blake2b(patternBytes(500, 81))}:${blake2b(patternBytes(900, 82))}`;
|
||||||
|
|
||||||
|
it("stores a file whose hash matches", async () => {
|
||||||
|
const plaintext = patternBytes(700, 80);
|
||||||
|
const t = setup(plaintext, { hash: blake2b(plaintext) });
|
||||||
|
|
||||||
|
await t.run();
|
||||||
|
|
||||||
|
expectSameBytes(readFileSync(t.outPath), plaintext);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a mismatch, stores nothing, names the file and does not retry", async () => {
|
||||||
|
const t = setup(patternBytes(700, 80), {
|
||||||
|
hash: blake2b(patternBytes(700, 79)),
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(t.run()).rejects.toThrow(
|
||||||
|
/file 999: content hash .* does not match/,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(readdirSync(t.dir)).toEqual([]);
|
||||||
|
expect(t.requests()).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stores a file with no recorded hash unchecked", async () => {
|
||||||
|
const plaintext = patternBytes(700, 80);
|
||||||
|
const t = setup(plaintext, { hash: undefined });
|
||||||
|
|
||||||
|
await t.run();
|
||||||
|
|
||||||
|
expectSameBytes(readFileSync(t.outPath), plaintext);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stores a live photo whose image and video hashes match", async () => {
|
||||||
|
const t = setup(livePhotoZip, {
|
||||||
|
fileType: "livePhoto",
|
||||||
|
hash: livePhotoHash,
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.run();
|
||||||
|
|
||||||
|
expectSameBytes(readFileSync(t.outPath), livePhotoZip);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("hashes a large live photo entry as it decompresses, never whole", async () => {
|
||||||
|
// 64 MiB of zeros deflates to a few kilobytes, the shape of a ZIP
|
||||||
|
// that would exhaust memory if expanded whole.
|
||||||
|
const image = new Uint8Array(64 * 1024 * 1024);
|
||||||
|
const video = patternBytes(900, 83);
|
||||||
|
const zip = zipSync({ "image.heic": image, "video.mov": video });
|
||||||
|
const t = setup(zip, {
|
||||||
|
fileType: "livePhoto",
|
||||||
|
hash: `${blake2b(image)}:${blake2b(video)}`,
|
||||||
|
});
|
||||||
|
|
||||||
|
await t.run();
|
||||||
|
|
||||||
|
expectSameBytes(readFileSync(t.outPath), zip);
|
||||||
|
const hashed = hashHook.lengths.reduce((a, b) => a + b, 0);
|
||||||
|
expect(hashed).toBe(image.length + video.length);
|
||||||
|
expect(Math.max(...hashHook.lengths)).toBeLessThanOrEqual(
|
||||||
|
2 * STREAM_CHUNK_SIZE,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a live photo whose hash does not match", async () => {
|
||||||
|
// The whole ZIP's hash is not the recorded one: each part is hashed.
|
||||||
|
const t = setup(livePhotoZip, {
|
||||||
|
fileType: "livePhoto",
|
||||||
|
hash: blake2b(livePhotoZip),
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(t.run()).rejects.toThrow(
|
||||||
|
/file 999: content hash .* does not match/,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(readdirSync(t.dir)).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -351,6 +351,46 @@ describe("model.decryptFile", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("reads the recorded content hash, joining an older live photo's two parts", () => {
|
||||||
|
const masterKey = sodium.crypto_secretbox_keygen();
|
||||||
|
const { collectionKey } = buildRawCollection(masterKey);
|
||||||
|
const hashOf = (metadata: Record<string, unknown>) =>
|
||||||
|
decryptFile(
|
||||||
|
buildRawFile(collectionKey, {
|
||||||
|
metadata: { title: "x", ...metadata },
|
||||||
|
}),
|
||||||
|
collectionKey,
|
||||||
|
).metadata.hash;
|
||||||
|
|
||||||
|
expect(hashOf({ fileType: 0, hash: "H" })).toBe("H");
|
||||||
|
expect(
|
||||||
|
hashOf({ fileType: 2, hash: "H", imageHash: "I", videoHash: "V" }),
|
||||||
|
).toBe("H");
|
||||||
|
expect(hashOf({ fileType: 2, imageHash: "I", videoHash: "V" })).toBe(
|
||||||
|
"I:V",
|
||||||
|
);
|
||||||
|
expect(hashOf({ fileType: 2, imageHash: "I" })).toBeUndefined();
|
||||||
|
expect(
|
||||||
|
hashOf({ fileType: 0, imageHash: "I", videoHash: "V" }),
|
||||||
|
).toBeUndefined();
|
||||||
|
expect(hashOf({ fileType: 0 })).toBeUndefined();
|
||||||
|
expect(hashOf({ fileType: 0, hash: 42 })).toBeUndefined();
|
||||||
|
expect(
|
||||||
|
hashOf({ fileType: 2, imageHash: "I", videoHash: 7 }),
|
||||||
|
).toBeUndefined();
|
||||||
|
// An empty string counts as absent, not as a hash to match.
|
||||||
|
expect(hashOf({ fileType: 0, hash: "" })).toBeUndefined();
|
||||||
|
expect(
|
||||||
|
hashOf({ fileType: 2, hash: "", imageHash: "I", videoHash: "V" }),
|
||||||
|
).toBe("I:V");
|
||||||
|
expect(
|
||||||
|
hashOf({ fileType: 2, imageHash: "", videoHash: "V" }),
|
||||||
|
).toBeUndefined();
|
||||||
|
expect(
|
||||||
|
hashOf({ fileType: 2, imageHash: "I", videoHash: "" }),
|
||||||
|
).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
it("maps fileType numbers to FileType strings", () => {
|
it("maps fileType numbers to FileType strings", () => {
|
||||||
// Ente uses: 0=image, 1=video, 2=livePhoto
|
// Ente uses: 0=image, 1=video, 2=livePhoto
|
||||||
const masterKey = sodium.crypto_secretbox_keygen();
|
const masterKey = sodium.crypto_secretbox_keygen();
|
||||||
|
|||||||
@@ -1069,6 +1069,11 @@ fastq@^1.6.0:
|
|||||||
dependencies:
|
dependencies:
|
||||||
reusify "^1.0.4"
|
reusify "^1.0.4"
|
||||||
|
|
||||||
|
fflate@0.8.3:
|
||||||
|
version "0.8.3"
|
||||||
|
resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.3.tgz#bc27d8eb30343d4d512abb03480202ce65d825fc"
|
||||||
|
integrity sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==
|
||||||
|
|
||||||
file-entry-cache@^8.0.0:
|
file-entry-cache@^8.0.0:
|
||||||
version "8.0.0"
|
version "8.0.0"
|
||||||
resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f"
|
resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f"
|
||||||
|
|||||||
Reference in New Issue
Block a user