sneak/quak
1
0
Fork 0
Code Issues 8 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Store live photos as their image and their video (closes #107)
check / check (push) Successful in 41s
Details

Browse Source 
A live photo, which Ente stores as one ZIP, is unpacked as it downloads
into its image and its video, each `<fileID>.<ext>` with its extension
from the ZIP, beside `<fileID>.livephoto.json`, which names the two.
Both are checked against the recorded hash and renamed into place only
when both are complete. The backup and the content cache count a live
photo as stored only with both files, album folders link both,
`quak get` writes both, and the content result gives the video as
`videoPath`. A ZIP an earlier version stored is replaced.

Model: opus-5-5
This commit is contained in:
clawbot
2026-09-23 12:50:02 +00:00
parent 04094a8cfb
commit a066134a17
15 changed files with 1340 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/download/download.test.ts
+155 -48
View File
@@ -55,6 +55,7 @@ import {
readFileSync,
rmSync,
mkdtempSync,
statSync,
writeFileSync,
} from "node:fs";
import { dirname, join } from "node:path";
@@ -81,6 +82,7 @@ import {
writeAtomic,
} from "../../src/download/index.js";
import type { EnteFile, FileMetadata } from "../../src/model/types.js";
import { IMAGE, livePhotoHash, livePhotoZip, VIDEO } from "../live-photo.js";
// ---------------------------------------------------------------------------
// Test helpers
@@ -106,6 +108,8 @@ import type { EnteFile, FileMetadata } from "../../src/model/types.js";
const renameHook = vi.hoisted(() => ({
calls: [] as { from: string; to: string; sourceExisted: boolean }[],
failWith: null as Error | null,
// When set, only a rename to this path fails.
failTo: null as string | null,
}));
/**
@@ -181,7 +185,10 @@ vi.mock("node:fs/promises", async (importOriginal) => {
sourceExisted: sourceExists(from),
});
durabilityHook.events.push(`rename:${to}`);
if (renameHook.failWith !== null) {
if (
renameHook.failWith !== null &&
(renameHook.failTo === null || renameHook.failTo === to)
) {
throw renameHook.failWith;
}
await actual.rename(from, to);
@@ -216,6 +223,7 @@ beforeEach(() => {
hashHook.lengths.length = 0;
renameHook.calls.length = 0;
renameHook.failWith = null;
renameHook.failTo = null;
durabilityHook.events.length = 0;
writeHook.writes.length = 0;
});
@@ -1642,38 +1650,35 @@ describe.each(entryPoints)("$name progress", ({ name, download }) => {
});
});
describe("downloadFile content hash", () => {
// Node's own BLAKE2b-512 is the reference, so these tests do not depend
// on the code under test to compute what they expect.
const blake2b = (bytes: Uint8Array): string =>
createHash("blake2b512").update(bytes).digest("base64");
// Node's own BLAKE2b-512 is the reference, so the tests below do not depend on
// the code under test to compute what they expect.
const blake2b = (bytes: Uint8Array): string =>
createHash("blake2b512").update(bytes).digest("base64");
// Serve `plaintext` encrypted as file 999 with the given metadata. Four
// responses are scripted so a retried mismatch would show in `requests`.
const setup = (plaintext: Uint8Array, metadata: Partial<FileMetadata>) => {
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(plaintext, key);
const file = buildMockEnteFile(key, header, header);
file.metadata = { ...file.metadata, ...metadata };
const body = { kind: "body", bytes: ciphertext } as const;
const { fetch, requests } = scriptedCdnFetch(body, body, body, body);
const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } });
const dir = mkdtempSync(join(testDir, "hash-"));
const outPath = join(dir, "f.bin");
return {
run: () => downloadFile(api, file, outPath),
dir,
outPath,
requests,
};
// Serve `plaintext` encrypted as file 999 with the given metadata, to be
// written to `f.bin` in a fresh directory. Four responses are scripted so a
// retried failure would show in `requests`.
const setup = (plaintext: Uint8Array, metadata: Partial<FileMetadata>) => {
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(plaintext, key);
const file = buildMockEnteFile(key, header, header);
file.metadata = { ...file.metadata, ...metadata };
const body = { kind: "body", bytes: ciphertext } as const;
const { fetch, requests } = scriptedCdnFetch(body, body, body, body);
const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } });
const dir = mkdtempSync(join(testDir, "hash-"));
const outPath = join(dir, "f.bin");
return {
run: () => downloadFile(api, file, outPath),
api,
file,
dir,
outPath,
requests,
};
};
const livePhotoZip = zipSync({
"image.heic": patternBytes(500, 81),
"video.mov": patternBytes(900, 82),
});
const livePhotoHash = `${blake2b(patternBytes(500, 81))}:${blake2b(patternBytes(900, 82))}`;
describe("downloadFile content hash", () => {
it("stores a file whose hash matches", async () => {
const plaintext = patternBytes(700, 80);
const t = setup(plaintext, { hash: blake2b(plaintext) });
@@ -1706,17 +1711,18 @@ describe("downloadFile content hash", () => {
});
it("stores a live photo whose image and video hashes match", async () => {
const t = setup(livePhotoZip, {
const t = setup(livePhotoZip(), {
fileType: "livePhoto",
hash: livePhotoHash,
hash: livePhotoHash(),
});
await t.run();
expectSameBytes(readFileSync(t.outPath), livePhotoZip);
expect(readFileSync(join(t.dir, "f.heic"))).toEqual(Buffer.from(IMAGE));
expect(readFileSync(join(t.dir, "f.mov"))).toEqual(Buffer.from(VIDEO));
});
it("hashes a large live photo entry as it decompresses, never whole", async () => {
it("writes a large live photo entry as it decompresses, never whole", async () => {
// 64 MiB of zeros deflates to a few kilobytes, the shape of a ZIP
// that would exhaust memory if expanded whole.
const image = new Uint8Array(64 * 1024 * 1024);
@@ -1724,40 +1730,45 @@ describe("downloadFile content hash", () => {
const zip = zipSync({ "image.heic": image, "video.mov": video });
const t = setup(zip, {
fileType: "livePhoto",
hash: `${blake2b(image)}:${blake2b(video)}`,
hash: livePhotoHash(image, video),
});
await t.run();
expectSameBytes(readFileSync(t.outPath), zip);
expect(statSync(join(t.dir, "f.heic")).size).toBe(image.length);
expectSameBytes(readFileSync(join(t.dir, "f.mov")), video);
const hashed = hashHook.lengths.reduce((a, b) => a + b, 0);
expect(hashed).toBe(image.length + video.length);
expect(Math.max(...hashHook.lengths)).toBeLessThanOrEqual(
2 * STREAM_CHUNK_SIZE,
);
const written = writeHook.writes.filter((w) => w.path.endsWith(".tmp"));
expect(Math.max(...written.map((w) => w.length))).toBeLessThanOrEqual(
2 * STREAM_CHUNK_SIZE,
);
});
it("rejects a live photo whose hash does not match", async () => {
it("rejects a live photo whose hash does not match, keeping what was there", async () => {
// The whole ZIP's hash is not the recorded one: each part is hashed.
const t = setup(livePhotoZip, {
fileType: "livePhoto",
hash: blake2b(livePhotoZip),
});
const zip = livePhotoZip();
const t = setup(zip, { fileType: "livePhoto", hash: blake2b(zip) });
writeFileSync(t.outPath, "an earlier download");
await expect(t.run()).rejects.toThrow(
/file 999: content hash .* does not match/,
);
expect(readdirSync(t.dir)).toEqual([]);
expect(readdirSync(t.dir)).toEqual(["f.bin"]);
expect(readFileSync(t.outPath, "utf-8")).toBe("an earlier download");
});
it("rejects a live photo that is not a readable ZIP and does not retry", async () => {
// Bytes 8-9 of a ZIP entry's local header name its compression
// method; 99 is one no reader knows, so the entry cannot be read.
const zip = livePhotoZip.slice();
const zip = livePhotoZip();
zip[8] = 99;
zip[9] = 0;
const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash });
const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() });
await expect(t.run()).rejects.toThrow(
/file 999: live photo is not a readable ZIP/,
@@ -1768,8 +1779,8 @@ describe("downloadFile content hash", () => {
});
it("rejects a live photo ZIP with no image entry", async () => {
const zip = zipSync({ "video.mov": patternBytes(900, 82) });
const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash });
const zip = livePhotoZip({ "video.mov": VIDEO });
const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() });
await expect(t.run()).rejects.toThrow(
/file 999: live photo ZIP does not hold both an image and a video/,
@@ -1779,8 +1790,8 @@ describe("downloadFile content hash", () => {
});
it("rejects a live photo ZIP with no video entry", async () => {
const zip = zipSync({ "image.heic": patternBytes(500, 81) });
const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash });
const zip = livePhotoZip({ "image.heic": IMAGE });
const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() });
await expect(t.run()).rejects.toThrow(
/file 999: live photo ZIP does not hold both an image and a video/,
@@ -1789,3 +1800,99 @@ describe("downloadFile content hash", () => {
expect(readdirSync(t.dir)).toEqual([]);
});
});
// ---------------------------------------------------------------------------
// Live photos
//
// A live photo arrives as a ZIP of its image and its video. It is written as
// those two files, which a photo viewer can open, each named after the
// destination with its own extension from the ZIP, the way Ente's clients name
// them when they save one.
// ---------------------------------------------------------------------------
describe("downloadFile live photos", () => {
const livePhoto = { fileType: "livePhoto", hash: livePhotoHash() } as const;
it("names the image and the video after the title when no outPath is given", async () => {
const t = setup(livePhotoZip(), {
...livePhoto,
title: "IMG_1234.HEIC",
});
const result = await inDirectory(t.dir, () =>
downloadFile(t.api, t.file),
);
// `bytesWritten` is the length of the decrypted ZIP.
expect(result).toEqual({
path: "IMG_1234.heic",
videoPath: "IMG_1234.mov",
bytesWritten: livePhotoZip().length,
});
expect(readdirSync(t.dir).sort()).toEqual([
"IMG_1234.heic",
"IMG_1234.mov",
]);
});
it("gives each part its own extension from the ZIP, letters and digits only", async () => {
const zip = livePhotoZip({ "image.JPG": IMAGE, "video.m-4v": VIDEO });
const t = setup(zip, livePhoto);
const result = await downloadFile(t.api, t.file, join(t.dir, "f.HEIC"));
expect(result.path).toBe(join(t.dir, "f.JPG"));
expect(result.videoPath).toBe(join(t.dir, "f.bin"));
expect(readdirSync(t.dir).sort()).toEqual(["f.JPG", "f.bin"]);
});
it("replaces what was at the destination, such as an earlier ZIP of the two", async () => {
const t = setup(livePhotoZip(), livePhoto);
writeFileSync(t.outPath, livePhotoZip());
await t.run();
expect(readdirSync(t.dir).sort()).toEqual(["f.heic", "f.mov"]);
});
it("renames the image and then the video into place, each from its own temp file", async () => {
const t = setup(livePhotoZip(), livePhoto);
await t.run();
expect(
renameHook.calls.map((c) => [
dirname(c.from),
c.to,
c.sourceExisted,
]),
).toEqual([
[t.dir, join(t.dir, "f.heic"), true],
[t.dir, join(t.dir, "f.mov"), true],
]);
});
it("stores neither part when the video cannot be renamed into place", async () => {
const t = setup(livePhotoZip(), livePhoto);
renameHook.failWith = new Error("simulated rename failure");
renameHook.failTo = join(t.dir, "f.mov");
await expect(t.run()).rejects.toThrow("simulated rename failure");
expect(readdirSync(t.dir)).toEqual([]);
});
it("refuses an image and a video with the same extension, storing nothing", async () => {
// On a file system that ignores case, the two would be one file.
const zip = livePhotoZip({ "image.mov": IMAGE, "video.MOV": VIDEO });
const t = setup(zip, livePhoto);
writeFileSync(t.outPath, "an earlier download");
await expect(t.run()).rejects.toThrow(
/file 999: live photo's image and video have the same extension/,
);
expect(readdirSync(t.dir)).toEqual(["f.bin"]);
expect(t.requests()).toBe(1);
});
});