Store live photos as their image and their video (closes #107)

A live photo, which Ente stores as one ZIP, is unpacked as it downloads
into its image and its video, each `<fileID>.<ext>` with its extension
from the ZIP, beside `<fileID>.livephoto.json`, which names the two.
Both are checked against the recorded hash and renamed into place only
when both are complete. The backup and the content cache count a live
photo as stored only with both files, album folders link both,
`quak get` writes both, and the content result gives the video as
`videoPath`. A ZIP an earlier version stored is replaced.

Model: opus-5-5

test/cli/backup.test.ts

@@ -53,6 +53,13 @@ import { Library } from "../../src/library/index.js";
 import type { ContentSource } from "../../src/library/content.js";
 import type { CollectionsPage, FilesPage } from "../../src/client.js";
 import type { Collection, EnteFile } from "../../src/model/types.js";
+import {
+    asLivePhoto,
+    cdnSource,
+    IMAGE,
+    livePhotoZip,
+    VIDEO,
+} from "../live-photo.js";
 
 // `open` and `rename` are wrapped to record, in order, every fsync and rename,
 // so a test can pin the sequence "fsync the temp file, rename, fsync the
@@ -740,6 +747,28 @@ describe("the refresh before a backup", () => {
     });
 });
 
+// Every entry under collections/, one level of directories deep, with each
+// symlink's target.
+const tree = (outDir: string): string[] => {
+    const lines: string[] = [];
+    const list = (dir: string, prefix: string): void => {
+        for (const name of readdirSync(dir).sort()) {
+            const path = join(dir, name);
+            const st = lstatSync(path);
+            if (st.isSymbolicLink()) {
+                lines.push(`${prefix}${name} -> ${readlinkSync(path)}`);
+            } else if (st.isDirectory() && prefix === "") {
+                lines.push(`${name}/`);
+                list(path, `${name}/`);
+            } else {
+                lines.push(`${prefix}${name}`);
+            }
+        }
+    };
+    list(join(outDir, "collections"), "");
+    return lines;
+};
+
 // The album folders under collections/, driven through `runBackup` with a
 // stand-in library whose albums a test changes between runs.
 describe("backup album folders", () => {
@@ -763,28 +792,6 @@ describe("backup album folders", () => {
         },
     });
 
-    // Every entry under collections/, one level of directories deep, with each
-    // symlink's target.
-    const tree = (outDir: string): string[] => {
-        const lines: string[] = [];
-        const list = (dir: string, prefix: string): void => {
-            for (const name of readdirSync(dir).sort()) {
-                const path = join(dir, name);
-                const st = lstatSync(path);
-                if (st.isSymbolicLink()) {
-                    lines.push(`${prefix}${name} -> ${readlinkSync(path)}`);
-                } else if (st.isDirectory() && prefix === "") {
-                    lines.push(`${name}/`);
-                    list(path, `${name}/`);
-                } else {
-                    lines.push(`${prefix}${name}`);
-                }
-            }
-        };
-        list(join(outDir, "collections"), "");
-        return lines;
-    };
-
     const albumID = (outDir: string, jsonName: string): number =>
         JSON.parse(readFileSync(join(outDir, "collections", jsonName), "utf-8"))
             .id;
@@ -1054,3 +1061,190 @@ describe("backup album folders", () => {
         ).toBe(json);
     });
 });
+
+// A live photo, which Ente stores as one ZIP, is backed up as its image and
+// its video, which a photo viewer can open, beside a JSON file naming them,
+// and its album folder links both. These tests download a live photo ZIP
+// through the real download layer (test/live-photo.ts).
+describe("backup of live photos", () => {
+    // An account of one album, Trip (10), holding `files`.
+    class TripClient extends MockClient {
+        constructor(private readonly files: EnteFile[]) {
+            super();
+        }
+        override async collectionsSince(): Promise<CollectionsPage> {
+            return {
+                collections: [collection(10, "Trip")],
+                deleted: [],
+                cursor: 1,
+            };
+        }
+        override async filesSince(): Promise<FilesPage> {
+            return { files: this.files, deleted: [], cursor: 1 };
+        }
+    }
+
+    const open = (files: EnteFile[], bodies: Map<number, Uint8Array>) =>
+        openLibrary(cdnSource(bodies), new TripClient(files));
+
+    // What an earlier version stored for live photo 500: the ZIP under the
+    // image's name, and its link.
+    const earlierZIP = (outDir: string): void => {
+        mkdirSync(join(outDir, "originals"), { recursive: true });
+        mkdirSync(join(outDir, "collections", "Trip"), { recursive: true });
+        writeFileSync(join(outDir, "originals", "500.HEIC"), livePhotoZip());
+        symlinkSync(
+            "../../originals/500.HEIC",
+            join(outDir, "collections", "Trip", "IMG_0500.HEIC"),
+        );
+    };
+
+    const stored = ["500.heic", "500.json", "500.livephoto.json", "500.mov"];
+    const linked = [
+        "Trip/",
+        "Trip/IMG_0500.heic -> ../../originals/500.heic",
+        "Trip/IMG_0500.mov -> ../../originals/500.mov",
+        "Trip.json",
+    ];
+
+    it("stores a live photo as its image and its video and links both", async () => {
+        const { file: live, body } = await asLivePhoto(
+            file(500, 10, "IMG_0500.HEIC"),
+        );
+        const lib = await open([live], new Map([[500, body]]));
+        const outDir = join(root, "backup");
+        const originals = join(outDir, "originals");
+
+        const result = await lib.backup({ downloadDirectory: outDir });
+
+        expect(result).toMatchObject({ downloaded: 1, failed: 0 });
+        expect(readdirSync(originals).sort()).toEqual(stored);
+        expect(readFileSync(join(originals, "500.heic"))).toEqual(
+            Buffer.from(IMAGE),
+        );
+        expect(readFileSync(join(originals, "500.mov"))).toEqual(
+            Buffer.from(VIDEO),
+        );
+        expect(tree(outDir)).toEqual(linked);
+
+        // Both parts are there, so the next run fetches nothing.
+        const second = await lib.backup({ downloadDirectory: outDir });
+        expect(second).toMatchObject({ downloaded: 0, skipped: 1, failed: 0 });
+        await lib.close();
+    });
+
+    it("gives both links of each live photo their own names when titles clash", async () => {
+        const a = await asLivePhoto(file(500, 10, "IMG_0001.HEIC"));
+        const b = await asLivePhoto(file(501, 10, "IMG_0001.HEIC"));
+        const lib = await open(
+            [a.file, b.file],
+            new Map([
+                [500, a.body],
+                [501, b.body],
+            ]),
+        );
+        const outDir = join(root, "backup");
+
+        await lib.backup({ downloadDirectory: outDir });
+
+        expect(tree(outDir)).toEqual([
+            "Trip/",
+            "Trip/IMG_0001 (500).heic -> ../../originals/500.heic",
+            "Trip/IMG_0001 (500).mov -> ../../originals/500.mov",
+            "Trip/IMG_0001 (501).heic -> ../../originals/501.heic",
+            "Trip/IMG_0001 (501).mov -> ../../originals/501.mov",
+            "Trip.json",
+        ]);
+        await lib.close();
+    });
+
+    it("replaces the ZIP an earlier version stored, and its link", async () => {
+        const { file: live, body } = await asLivePhoto(
+            file(500, 10, "IMG_0500.HEIC"),
+        );
+        const outDir = join(root, "backup");
+        earlierZIP(outDir);
+        const lib = await open([live], new Map([[500, body]]));
+
+        const result = await lib.backup({ downloadDirectory: outDir });
+
+        expect(result).toMatchObject({ downloaded: 1, failed: 0 });
+        expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored);
+        expect(tree(outDir)).toEqual(linked);
+        await lib.close();
+    });
+
+    it("stores nothing for a live photo that fails its hash, and keeps what was there", async () => {
+        const { file: live, body } = await asLivePhoto(
+            file(500, 10, "IMG_0500.HEIC"),
+            livePhotoZip(),
+            "not:the recorded hash",
+        );
+        const outDir = join(root, "backup");
+        earlierZIP(outDir);
+        const lib = await open([live], new Map([[500, body]]));
+
+        const result = await lib.backup({ downloadDirectory: outDir });
+
+        expect(result).toMatchObject({ downloaded: 0, failed: 1 });
+        expect(result.errors.map((e) => e.fileID)).toEqual([500]);
+        expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]);
+        expect(readdirSync(join(outDir, "originals"))).toEqual(["500.HEIC"]);
+        expect(tree(outDir)).toEqual([
+            "Trip/",
+            "Trip/IMG_0500.HEIC -> ../../originals/500.HEIC",
+            "Trip.json",
+        ]);
+        await lib.close();
+    });
+
+    it("copies both parts of a live photo the cache already holds", async () => {
+        const { file: live, body } = await asLivePhoto(
+            file(500, 10, "IMG_0500.HEIC"),
+        );
+        const lib = await open([live], new Map([[500, body]]));
+        const cached = await lib.photos.byID({ fileID: 500 })!.original();
+        const outDir = join(root, "backup");
+
+        const result = await lib.backup({ downloadDirectory: outDir });
+
+        expect(result).toMatchObject({ downloaded: 1, failed: 0 });
+        expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored);
+        expect(readFileSync(join(outDir, "originals", "500.mov"))).toEqual(
+            Buffer.from(VIDEO),
+        );
+        expect(tree(outDir)).toEqual(linked);
+        // The cache keeps its own copy.
+        expect(existsSync(cached.videoPath!)).toBe(true);
+        await lib.close();
+    });
+
+    it("serves a live photo the backup stored to a library reading the backup", async () => {
+        const { file: live, body } = await asLivePhoto(
+            file(500, 10, "IMG_0500.HEIC"),
+        );
+        const lib = await open([live], new Map([[500, body]]));
+        const outDir = join(root, "backup");
+        await lib.backup({ downloadDirectory: outDir });
+        await lib.close();
+
+        // Another cache over the same backup, whose server has nothing.
+        const reader = await Library.open({
+            client: new TripClient([live]),
+            cacheDirectory: join(root, "other-cache"),
+            downloadDirectory: outDir,
+            contentSource: cdnSource(new Map()),
+            refreshIntervalSeconds: 3600,
+            precacheThumbnails: false,
+            precacheOriginals: false,
+        });
+        const read = await reader.photos.byID({ fileID: 500 })!.original();
+
+        expect(read).toEqual({
+            path: join(outDir, "originals", "500.heic"),
+            videoPath: join(outDir, "originals", "500.mov"),
+            bytes: IMAGE.length,
+        });
+        await reader.close();
+    });
+});