quak/test/cli/metadata-backup.test.ts

/**
 * Tests for `quak backup-metadata <dir>`.
 *
 * This command dumps all decrypted account metadata into a directory
 * tree of plain JSON files, without downloading any file content (unless
 * `--exif` is given). It is fast and produces a complete plaintext record of
 * every collection name, file title, creation date, GPS coordinate, camera
 * model, caption, face label, and any other metadata the Ente clients have
 * attached.
 *
 * As of issue #52 it runs on the library API: `runMetadataBackup(lib, client,
 * dir)` enumerates collections and files from the library's cache rather than
 * scanning the client directly, and `--exif` reads each original through the
 * library's content cache (`photo.original()`). The ML fetch is unchanged. The
 * output tree is identical:
 *
 *     <dir>/
 *         account.json                { email, userID }
 *         collections/
 *             <id>-<sanitized-name>/
 *                 _collection.json    { id, name, type, magicMetadata?, ... }
 *                 <fileID>.json       { id, metadata, magicMetadata?, pubMagicMetadata? }
 *
 * The test builds a mock server with two collections, each with files
 * that have different combinations of metadata layers, and verifies
 * the output tree is correct and complete.
 */

import { gzipSync } from "node:zlib";
import {
    existsSync,
    mkdtempSync,
    readFileSync,
    readdirSync,
    rmSync,
} from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import sodium from "libsodium-wrappers-sumo";
import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
import { init, toBase64, encryptBlob } from "../../src/crypto/index.js";
import * as jpegJs from "jpeg-js";
import { Client } from "../../src/client.js";
import { Library } from "../../src/library/index.js";
import {
    runMetadataBackup,
    type MetadataBackupOptions,
} from "../../src/metadata-backup.js";
import { backupMetadataCommand } from "../../src/cli-commands.js";
import {
    buildAccount,
    buildMetaFetch,
    encryptSecretbox,
    encryptStreamBlob,
    type MetaMockState,
} from "../ente-server.js";

// One file per ML data request, so the two files of the mock account are
// fetched in two requests and one of them can fail on its own.
vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({
    ...(await importOriginal<typeof import("../../src/mldata-fetch.js")>()),
    MLDATA_BATCH_SIZE: 1,
}));

const TEST_EMAIL = "metabackup@example.com";
const TEST_PASSWORD = "metapass";

let mock: MetaMockState;
let testDir: string;

const buildMetaMock = async (): Promise<MetaMockState> => {
    const account = await buildAccount(TEST_PASSWORD);
    const masterKey = account.masterKey;

    // Collection 1: "Vacation" with collection-level pubMagicMetadata
    const ck1 = sodium.crypto_secretbox_keygen();
    const { ciphertext: encCK1, nonce: ck1N } = encryptSecretbox(
        ck1,
        masterKey,
    );
    const { ciphertext: encCN1, nonce: cn1N } = encryptSecretbox(
        new TextEncoder().encode("Vacation"),
        ck1,
    );
    const collPubMagic = JSON.stringify({ coverID: 999, sortBy: "date" });
    const { ciphertext: encCollPM, header: collPMHeader } = encryptStreamBlob(
        new TextEncoder().encode(collPubMagic),
        ck1,
    );

    const rawColl1 = {
        id: 10,
        owner: { id: 42 },
        encryptedKey: toBase64(encCK1),
        keyDecryptionNonce: toBase64(ck1N),
        encryptedName: toBase64(encCN1),
        nameDecryptionNonce: toBase64(cn1N),
        type: "album",
        updationTime: 1700000000000000,
        pubMagicMetadata: {
            version: 1,
            count: 1,
            data: toBase64(encCollPM),
            header: toBase64(collPMHeader),
        },
    };

    // Collection 2: "../Work" with no magic metadata. The server chose a name
    // that tries to climb out of the backup directory.
    const ck2 = sodium.crypto_secretbox_keygen();
    const { ciphertext: encCK2, nonce: ck2N } = encryptSecretbox(
        ck2,
        masterKey,
    );
    const { ciphertext: encCN2, nonce: cn2N } = encryptSecretbox(
        new TextEncoder().encode("../Work"),
        ck2,
    );
    const rawColl2 = {
        id: 20,
        owner: { id: 42 },
        encryptedKey: toBase64(encCK2),
        keyDecryptionNonce: toBase64(ck2N),
        encryptedName: toBase64(encCN2),
        nameDecryptionNonce: toBase64(cn2N),
        type: "folder",
        updationTime: 1700000000000000,
    };

    // File 100 in coll 10: has metadata + pubMagicMetadata
    const fk1 = sodium.crypto_secretstream_xchacha20poly1305_keygen();
    const { ciphertext: encFK1, nonce: fk1N } = encryptSecretbox(fk1, ck1);
    const meta1 = JSON.stringify({
        title: "beach.jpg",
        fileType: 0,
        creationTime: 1700000000000000,
        modificationTime: 1700000000000000,
        latitude: 35.6762,
        longitude: 139.6503,
    });
    const { ciphertext: encMeta1, header: meta1Header } = encryptStreamBlob(
        new TextEncoder().encode(meta1),
        fk1,
    );
    const pubMagic1 = JSON.stringify({
        w: 3000,
        h: 2000,
        cameraMake: "SONY",
        cameraModel: "DSC-RX1RM3",
    });
    const { ciphertext: encPM1, header: pm1Header } = encryptStreamBlob(
        new TextEncoder().encode(pubMagic1),
        fk1,
    );

    const rawFile1 = {
        id: 100,
        collectionID: 10,
        ownerID: 42,
        encryptedKey: toBase64(encFK1),
        keyDecryptionNonce: toBase64(fk1N),
        metadata: {
            encryptedData: toBase64(encMeta1),
            decryptionHeader: toBase64(meta1Header),
        },
        pubMagicMetadata: {
            version: 1,
            count: 1,
            data: toBase64(encPM1),
            header: toBase64(pm1Header),
        },
        file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
        thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
        updationTime: 1700000000000000,
    };

    // File 200 in coll 20: metadata only, no magic metadata
    const fk2 = sodium.crypto_secretstream_xchacha20poly1305_keygen();
    const { ciphertext: encFK2, nonce: fk2N } = encryptSecretbox(fk2, ck2);
    const meta2 = JSON.stringify({
        title: "diagram.png",
        fileType: 0,
        creationTime: 1710000000000000,
        modificationTime: 1710000000000000,
    });
    const { ciphertext: encMeta2, header: meta2Header } = encryptStreamBlob(
        new TextEncoder().encode(meta2),
        fk2,
    );

    const rawFile2 = {
        id: 200,
        collectionID: 20,
        ownerID: 42,
        encryptedKey: toBase64(encFK2),
        keyDecryptionNonce: toBase64(fk2N),
        metadata: {
            encryptedData: toBase64(encMeta2),
            decryptionHeader: toBase64(meta2Header),
        },
        file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
        thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
        updationTime: 1710000000000000,
    };

    // Encrypt ML data for file 100 (gzipped JSON, encrypted with file key)
    const mlPayload = JSON.stringify({
        face: {
            version: 1,
            client: "test",
            width: 3000,
            height: 2000,
            faces: [
                {
                    faceID: "face-abc",
                    detection: {
                        box: { x: 0.1, y: 0.2, width: 0.3, height: 0.4 },
                        landmarks: [
                            { x: 0.15, y: 0.25 },
                            { x: 0.25, y: 0.25 },
                        ],
                    },
                    score: 0.98,
                    blur: 12.5,
                    embedding: [0.1, 0.2, 0.3],
                },
            ],
        },
        clip: {
            version: 1,
            client: "test",
            embedding: [0.5, 0.6, 0.7],
        },
    });
    const gzipped = gzipSync(Buffer.from(mlPayload));
    const { header: mlHeader, ciphertext: mlCiphertext } = encryptBlob(
        new Uint8Array(gzipped),
        fk1,
    );

    // Generate a real JPEG for EXIF extraction tests
    const jw = 100;
    const jh = 80;
    const jpixels = new Uint8Array(jw * jh * 4);
    for (let i = 0; i < jpixels.length; i += 4) {
        jpixels[i] = 255;
        jpixels[i + 1] = 0;
        jpixels[i + 2] = 0;
        jpixels[i + 3] = 255;
    }
    const tinyJpeg = jpegJs.encode(
        { data: jpixels, width: jw, height: jh },
        80,
    ).data;
    const filePush1 =
        sodium.crypto_secretstream_xchacha20poly1305_init_push(fk1);
    const encFileBody1 = sodium.crypto_secretstream_xchacha20poly1305_push(
        filePush1.state,
        new Uint8Array(tinyJpeg),
        null,
        sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
    );
    // Patch rawFile1's file.decryptionHeader to match the push header
    rawFile1.file.decryptionHeader = toBase64(filePush1.header);

    return {
        verifier: account.verifier,
        srpAttributes: account.srpAttributes,
        keyAttributes: account.keyAttributes,
        encryptedToken: account.encryptedToken,
        collections: [rawColl1, rawColl2],
        filesByCollection: { 10: [rawFile1], 20: [rawFile2] },
        encryptedMLData: {
            100: {
                encryptedData: toBase64(mlCiphertext),
                decryptionHeader: toBase64(mlHeader),
            },
        },
        fileCiphertexts: { 100: encFileBody1 },
    };
};

beforeAll(async () => {
    await init();
    await sodium.ready;
    mock = await buildMetaMock();
    testDir = mkdtempSync(join(tmpdir(), "quak-meta-backup-test-"));
});

afterAll(() => {
    if (testDir && existsSync(testDir))
        rmSync(testDir, { recursive: true, force: true });
});

// Log in against the mock and open a library over its cache. The point commands
// open the library with the background precache off and a long refresh interval;
// the same here keeps the test deterministic (no thumbnail/original prefetch it
// did not ask for, no second refresh mid-test).
const openLib = async (client: Client): Promise<Library> =>
    Library.open({
        // The library client omits `fetchMLData`, matching how the CLI opens
        // point commands: `runMetadataBackup` fetches ML data itself through
        // the client, so the library's background backfill would only be a
        // redundant second pass over the same endpoint.
        client: {
            whoami: () => client.whoami(),
            collectionsSince: (args) => client.collectionsSince(args),
            filesSince: (args) => client.filesSince(args),
            contentSource: () => client.contentSource(),
        },
        cacheDirectory: mkdtempSync(join(testDir, "cache-")),
        refreshIntervalSeconds: 3600,
        precacheThumbnails: false,
        precacheOriginals: false,
    });

// Run one metadata backup end to end: fresh client, fresh library, then close.
const runBackup = async (
    outDir: string,
    opts?: MetadataBackupOptions,
): Promise<void> => {
    const client = await Client.login({
        email: TEST_EMAIL,
        password: TEST_PASSWORD,
        apiOptions: { fetch: buildMetaFetch(mock) },
    });
    const lib = await openLib(client);
    try {
        await runMetadataBackup(lib, client, outDir, opts);
    } finally {
        lib.close();
    }
};

describe("quak backup-metadata", () => {
    it("writes account.json with email and userID", async () => {
        const outDir = join(testDir, "full");
        await runBackup(outDir);

        const account = JSON.parse(
            readFileSync(join(outDir, "account.json"), "utf-8"),
        );
        expect(account.email).toBe(TEST_EMAIL);
        expect(account.userID).toBe(42);
    });

    it("creates per-collection directories with _collection.json", async () => {
        const outDir = join(testDir, "collections");
        await runBackup(outDir);

        const collDirs = readdirSync(join(outDir, "collections"));
        // "../Work" is sanitized into one directory name.
        expect(collDirs.sort()).toEqual(["10-Vacation", "20-__Work"]);

        // Find the Vacation collection dir (prefixed with ID)
        const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
        expect(vacDir).toBeDefined();
        const collMeta = JSON.parse(
            readFileSync(
                join(outDir, "collections", vacDir, "_collection.json"),
                "utf-8",
            ),
        );
        expect(collMeta.id).toBe(10);
        expect(collMeta.name).toBe("Vacation");
        expect(collMeta.type).toBe("album");
    });

    it("decrypts collection-level pubMagicMetadata", async () => {
        const outDir = join(testDir, "coll-magic");
        await runBackup(outDir);

        const collDirs = readdirSync(join(outDir, "collections"));
        const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
        const collMeta = JSON.parse(
            readFileSync(
                join(outDir, "collections", vacDir, "_collection.json"),
                "utf-8",
            ),
        );
        expect(collMeta.pubMagicMetadata).toBeDefined();
        expect(collMeta.pubMagicMetadata.coverID).toBe(999);
        expect(collMeta.pubMagicMetadata.sortBy).toBe("date");
    });

    it("writes per-file JSON with all three metadata layers", async () => {
        const outDir = join(testDir, "file-meta");
        await runBackup(outDir);

        const collDirs = readdirSync(join(outDir, "collections"));
        const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
        const fileMeta = JSON.parse(
            readFileSync(
                join(outDir, "collections", vacDir, "100.json"),
                "utf-8",
            ),
        );
        expect(fileMeta.id).toBe(100);
        expect(fileMeta.metadata.title).toBe("beach.jpg");
        expect(fileMeta.metadata.latitude).toBeCloseTo(35.6762);
        expect(fileMeta.pubMagicMetadata.cameraMake).toBe("SONY");
        expect(fileMeta.pubMagicMetadata.w).toBe(3000);
    });

    it("handles files with no magic metadata gracefully", async () => {
        const outDir = join(testDir, "no-magic");
        await runBackup(outDir);

        const collDirs = readdirSync(join(outDir, "collections"));
        const workDir = collDirs.find((d) => d.includes("Work"))!;
        const fileMeta = JSON.parse(
            readFileSync(
                join(outDir, "collections", workDir, "200.json"),
                "utf-8",
            ),
        );
        expect(fileMeta.id).toBe(200);
        expect(fileMeta.metadata.title).toBe("diagram.png");
        expect(fileMeta.pubMagicMetadata).toBeUndefined();
        expect(fileMeta.magicMetadata).toBeUndefined();
    });

    it("is incremental: second run does not fail", async () => {
        const outDir = join(testDir, "incremental");
        await runBackup(outDir);
        await runBackup(outDir);

        const account = JSON.parse(
            readFileSync(join(outDir, "account.json"), "utf-8"),
        );
        expect(account.email).toBe(TEST_EMAIL);
    });

    it("fetches and decrypts ML data by default", async () => {
        const outDir = join(testDir, "ml-data");
        await runBackup(outDir);

        const collDirs = readdirSync(join(outDir, "collections"));
        const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
        const fileMeta = JSON.parse(
            readFileSync(
                join(outDir, "collections", vacDir, "100.json"),
                "utf-8",
            ),
        );

        // ML data should be present and decrypted
        expect(fileMeta.mlData).toBeDefined();
        expect(fileMeta.mlData.face).toBeDefined();
        expect(fileMeta.mlData.face.faces.length).toBe(1);
        expect(fileMeta.mlData.face.faces[0].faceID).toBe("face-abc");
        expect(fileMeta.mlData.face.faces[0].score).toBeCloseTo(0.98);
        expect(fileMeta.mlData.face.faces[0].detection.box.x).toBeCloseTo(0.1);
        expect(fileMeta.mlData.clip).toBeDefined();
        expect(fileMeta.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
    });

    it("extracts EXIF from downloaded files when --exif is set", async () => {
        const outDir = join(testDir, "exif-data");
        await runBackup(outDir, { exif: true });

        const collDirs = readdirSync(join(outDir, "collections"));
        const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
        const fileMeta = JSON.parse(
            readFileSync(
                join(outDir, "collections", vacDir, "100.json"),
                "utf-8",
            ),
        );

        // imageMetadata from JPEG parsing should be present
        expect(fileMeta.imageMetadata).toBeDefined();
        expect(fileMeta.imageMetadata.format).toBe("jpeg");
        expect(fileMeta.imageMetadata.width).toBe(100);
        expect(fileMeta.imageMetadata.height).toBe(80);
        expect(fileMeta.imageMetadataError).toBeUndefined();

        // File 200 has no original on the mock server, so extraction fails
        // and the reason is recorded instead of the field being left out.
        const workDir = collDirs.find((d) => d.includes("Work"))!;
        const failedMeta = JSON.parse(
            readFileSync(
                join(outDir, "collections", workDir, "200.json"),
                "utf-8",
            ),
        );
        expect(failedMeta.imageMetadata).toBeUndefined();
        expect(failedMeta.imageMetadataError).toEqual(expect.any(String));
    });
});

describe("quak backup-metadata when a file's ML data cannot be read", () => {
    // Run the CLI command against `fetch`, the mock unless given, and return
    // its exit code, stderr and output directory.
    const runCommand = async (fetch = buildMetaFetch(mock)) => {
        const client = await Client.login({
            email: TEST_EMAIL,
            password: TEST_PASSWORD,
            apiOptions: {
                fetch,
                retry: { sleep: async () => {} },
            },
        });
        const outDir = mkdtempSync(join(testDir, "ml-fail-"));
        let stderr = "";
        const code = await backupMetadataCommand(
            {
                stdout: { write: () => true },
                stderr: { write: (text: string) => (stderr += text) },
                sessionDir: testDir,
                cacheDir: mkdtempSync(join(testDir, "cache-")),
                loadSession: () => client,
            },
            outDir,
            {},
        );
        return { code, stderr, outDir };
    };

    it("writes every file, marks the failed batch's files, and exits 1", async () => {
        const { code, stderr, outDir } = await runCommand(
            buildMetaFetch(mock, 200),
        );

        expect(code).toBe(1);
        expect(stderr).toContain("ML data request for 1 file(s) failed");

        const ok = JSON.parse(
            readFileSync(
                join(outDir, "collections", "10-Vacation", "100.json"),
                "utf-8",
            ),
        );
        expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
        expect(ok.mlDataError).toBeUndefined();

        const failed = JSON.parse(
            readFileSync(
                join(outDir, "collections", "20-__Work", "200.json"),
                "utf-8",
            ),
        );
        expect(failed.metadata.title).toBe("diagram.png");
        expect(failed.mlData).toBeUndefined();
        expect(failed.mlDataError).toContain("500");
    });

    // The server sends ML data for diagram.png encrypted under a key that is
    // not the file's, so quak cannot decrypt it.
    it("writes the reason for ML data that does not decrypt, and exits 1", async () => {
        const otherKey = sodium.crypto_secretstream_xchacha20poly1305_keygen();
        const { header, ciphertext } = encryptBlob(
            new Uint8Array(gzipSync(Buffer.from("{}"))),
            otherKey,
        );
        const { code, stderr, outDir } = await runCommand(
            buildMetaFetch({
                ...mock,
                encryptedMLData: {
                    ...mock.encryptedMLData,
                    200: {
                        encryptedData: toBase64(ciphertext),
                        decryptionHeader: toBase64(header),
                    },
                },
            }),
        );

        expect(code).toBe(1);
        const reason = "secretstream chunk authentication failed";
        expect(stderr).toContain(
            `ML data of file 200 could not be read: ${reason}\n`,
        );
        const failed = JSON.parse(
            readFileSync(
                join(outDir, "collections", "20-__Work", "200.json"),
                "utf-8",
            ),
        );
        expect(failed.mlData).toBeUndefined();
        expect(failed.mlDataError).toBe(reason);
        const ok = JSON.parse(
            readFileSync(
                join(outDir, "collections", "10-Vacation", "100.json"),
                "utf-8",
            ),
        );
        expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
    });

    it("exits 0 when every ML data request succeeds", async () => {
        const { code } = await runCommand();
        expect(code).toBe(0);
    });
});