Add quak backup-metadata: dump all decrypted metadata to plain JSON

New command: quak backup-metadata <dir>

Dumps every piece of decrypted account metadata into a directory tree
of plain JSON files without downloading any file content. Layout:

    <dir>/
        account.json                    { email, userID }
        collections/
            <id>-<name>/
                _collection.json        { id, name, type, pubMagicMetadata?, ... }
                <fileID>.json           { id, metadata, magicMetadata?, pubMagicMetadata? }

Also adds collection-level magic metadata decryption (magicMetadata,
pubMagicMetadata, sharedMagicMetadata) to decryptCollection, which was
previously only done for files. The server sends these for visibility
settings, sort order, cover photo selection, etc.

6 new tests covering: account.json, per-collection dirs with
_collection.json, collection pubMagicMetadata decryption, per-file
JSON with all three metadata layers, graceful handling of files with
no magic metadata, and incremental re-run safety. 116 total.

bin/quak.ts

@@ -9,6 +9,7 @@ import envPaths from "env-paths";
 import { Client, type ClientSnapshot } from "../src/client.js";
 import { init } from "../src/crypto/index.js";
 import { runBackup } from "../src/backup.js";
+import { runMetadataBackup } from "../src/metadata-backup.js";
 import {
     listMissingThumbnails,
     fixMissingThumbnails,
@@ -333,6 +334,18 @@ program
         },
     );
 
+program
+    .command("backup-metadata")
+    .description(
+        "Dump all decrypted account metadata (no file content) to a directory",
+    )
+    .argument("<dir>", "Output directory")
+    .action(async (dir: string) => {
+        await init();
+        const client = requireSession();
+        await runMetadataBackup(client, dir, (msg) => stderr.write(msg + "\n"));
+    });
+
 program
     .command("backup")
     .description(

src/metadata-backup.ts

@@ -0,0 +1,77 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { Client } from "./client.js";
+
+export type ProgressCallback = (message: string) => void;
+
+const sanitizePath = (name: string): string =>
+    name.replace(/[/\\:*?"<>|]/g, "_").replace(/^\.+/, "_");
+
+export const runMetadataBackup = async (
+    client: Client,
+    outDir: string,
+    onProgress?: ProgressCallback,
+): Promise<void> => {
+    const log = onProgress ?? (() => {});
+
+    mkdirSync(outDir, { recursive: true });
+    mkdirSync(join(outDir, "collections"), { recursive: true });
+
+    const { email, userID } = client.whoami();
+    writeFileSync(
+        join(outDir, "account.json"),
+        JSON.stringify({ email, userID }, null, 2),
+    );
+
+    log("Fetching collections...");
+    const collections = await client.listCollections();
+
+    for (const col of collections) {
+        const dirName = `${col.id}-${sanitizePath(col.name || "unnamed")}`;
+        const colDir = join(outDir, "collections", dirName);
+        mkdirSync(colDir, { recursive: true });
+
+        const collectionMeta: Record<string, unknown> = {
+            id: col.id,
+            name: col.name,
+            type: col.type,
+            ownerID: col.ownerID,
+            isShared: col.isShared,
+            updationTime: col.updationTime,
+        };
+        if (col.magicMetadata) collectionMeta.magicMetadata = col.magicMetadata;
+        if (col.pubMagicMetadata)
+            collectionMeta.pubMagicMetadata = col.pubMagicMetadata;
+        if (col.sharedMagicMetadata)
+            collectionMeta.sharedMagicMetadata = col.sharedMagicMetadata;
+
+        writeFileSync(
+            join(colDir, "_collection.json"),
+            JSON.stringify(collectionMeta, null, 2),
+        );
+
+        log(`[${col.name}] Fetching files...`);
+        const files = await client.listFiles(col.id, col.key);
+        log(`[${col.name}] ${files.length} file(s)`);
+
+        for (const file of files) {
+            const fileMeta: Record<string, unknown> = {
+                id: file.id,
+                collectionID: file.collectionID,
+                ownerID: file.ownerID,
+                metadata: file.metadata,
+                updationTime: file.updationTime,
+            };
+            if (file.magicMetadata) fileMeta.magicMetadata = file.magicMetadata;
+            if (file.pubMagicMetadata)
+                fileMeta.pubMagicMetadata = file.pubMagicMetadata;
+
+            writeFileSync(
+                join(colDir, `${file.id}.json`),
+                JSON.stringify(fileMeta, null, 2),
+            );
+        }
+    }
+
+    log("Metadata backup complete.");
+};

src/model/decrypt.ts

@@ -57,6 +57,9 @@ export const decryptCollection = (
         type: parseCollectionType(raw.type),
         updationTime: raw.updationTime,
         isShared: currentUserID !== undefined && raw.owner.id !== currentUserID,
+        magicMetadata: decryptMagicMetadata(raw.magicMetadata, key),
+        pubMagicMetadata: decryptMagicMetadata(raw.pubMagicMetadata, key),
+        sharedMagicMetadata: decryptMagicMetadata(raw.sharedMagicMetadata, key),
     };
 };
 

src/model/types.ts

@@ -15,6 +15,9 @@ export interface Collection {
     type: CollectionType;
     updationTime: Microseconds;
     isShared: boolean;
+    magicMetadata?: Record<string, unknown>;
+    pubMagicMetadata?: Record<string, unknown>;
+    sharedMagicMetadata?: Record<string, unknown>;
 }
 
 export type FileType = "image" | "video" | "livePhoto" | "unknown";
@@ -59,6 +62,9 @@ export interface RawCollection {
     type: string;
     updationTime: number;
     isDeleted?: boolean;
+    magicMetadata?: RawMagicMetadata;
+    pubMagicMetadata?: RawMagicMetadata;
+    sharedMagicMetadata?: RawMagicMetadata;
 }
 
 export interface RawMagicMetadata {

test/cli/metadata-backup.test.ts

@@ -0,0 +1,468 @@
+/**
+ * Tests for `quak backup-metadata <dir>`.
+ *
+ * This command dumps all decrypted account metadata into a directory
+ * tree of plain JSON files, without downloading any file content. It
+ * is fast (no multi-megabyte downloads) and produces a complete
+ * plaintext record of every collection name, file title, creation
+ * date, GPS coordinate, camera model, caption, face label, and any
+ * other metadata the Ente clients have attached.
+ *
+ * Layout:
+ *
+ *     <dir>/
+ *         account.json                { email, userID }
+ *         collections/
+ *             <id>-<sanitized-name>/
+ *                 _collection.json    { id, name, type, magicMetadata?, ... }
+ *                 <fileID>.json       { id, metadata, magicMetadata?, pubMagicMetadata? }
+ *
+ * The test builds a mock server with two collections, each with files
+ * that have different combinations of metadata layers, and verifies
+ * the output tree is correct and complete.
+ */
+
+import {
+    existsSync,
+    mkdtempSync,
+    readFileSync,
+    readdirSync,
+    rmSync,
+} from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import sodium from "libsodium-wrappers-sumo";
+import { SRP, SrpServer } from "fast-srp-hap";
+import { beforeAll, afterAll, describe, expect, it } from "vitest";
+import {
+    init,
+    toBase64,
+    deriveKEK,
+    deriveLoginSubkey,
+} from "../../src/crypto/index.js";
+import { Client } from "../../src/client.js";
+import { runMetadataBackup } from "../../src/metadata-backup.js";
+import type { KeyAttributes } from "../../src/auth/types.js";
+
+const TEST_EMAIL = "metabackup@example.com";
+const TEST_PASSWORD = "metapass";
+const TEST_OPS = 2;
+const TEST_MEM = 64 * 1024 * 1024;
+
+interface MetaMockState {
+    verifier: Buffer;
+    srpAttributes: Record<string, unknown>;
+    keyAttributes: KeyAttributes;
+    encryptedToken: string;
+    collections: Record<string, unknown>[];
+    filesByCollection: Record<number, Record<string, unknown>[]>;
+}
+
+let mock: MetaMockState;
+let testDir: string;
+
+const encryptSecretbox = (plaintext: Uint8Array, key: Uint8Array) => {
+    const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
+    const ciphertext = sodium.crypto_secretbox_easy(plaintext, nonce, key);
+    return { ciphertext, nonce };
+};
+
+const encryptStreamBlob = (plaintext: Uint8Array, key: Uint8Array) => {
+    const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
+    const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push(
+        push.state,
+        plaintext,
+        null,
+        sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
+    );
+    return { ciphertext, header: push.header };
+};
+
+const buildMetaMock = async (): Promise<MetaMockState> => {
+    const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
+    const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM);
+    const loginSubKeyBytes = deriveLoginSubkey(kek);
+    const srpUserID = "meta-srp";
+    const srpSalt = sodium.randombytes_buf(16);
+    const verifier = SRP.computeVerifier(
+        SRP.params["4096"],
+        Buffer.from(srpSalt),
+        Buffer.from(srpUserID),
+        Buffer.from(loginSubKeyBytes),
+    );
+
+    const masterKey = sodium.randombytes_buf(32);
+    const { ciphertext: encMK, nonce: mkNonce } = encryptSecretbox(
+        masterKey,
+        kek,
+    );
+    const kp = sodium.crypto_box_keypair();
+    const { ciphertext: encSK, nonce: skNonce } = encryptSecretbox(
+        kp.privateKey,
+        masterKey,
+    );
+    const tokenBytes = sodium.randombytes_buf(32);
+    const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey);
+
+    const keyAttributes: KeyAttributes = {
+        kekSalt: toBase64(kekSalt),
+        encryptedKey: toBase64(encMK),
+        keyDecryptionNonce: toBase64(mkNonce),
+        publicKey: toBase64(kp.publicKey),
+        encryptedSecretKey: toBase64(encSK),
+        secretKeyDecryptionNonce: toBase64(skNonce),
+        memLimit: TEST_MEM,
+        opsLimit: TEST_OPS,
+    };
+
+    // Collection 1: "Vacation" with collection-level pubMagicMetadata
+    const ck1 = sodium.crypto_secretbox_keygen();
+    const { ciphertext: encCK1, nonce: ck1N } = encryptSecretbox(
+        ck1,
+        masterKey,
+    );
+    const { ciphertext: encCN1, nonce: cn1N } = encryptSecretbox(
+        new TextEncoder().encode("Vacation"),
+        ck1,
+    );
+    const collPubMagic = JSON.stringify({ coverID: 999, sortBy: "date" });
+    const { ciphertext: encCollPM, header: collPMHeader } = encryptStreamBlob(
+        new TextEncoder().encode(collPubMagic),
+        ck1,
+    );
+
+    const rawColl1 = {
+        id: 10,
+        owner: { id: 42 },
+        encryptedKey: toBase64(encCK1),
+        keyDecryptionNonce: toBase64(ck1N),
+        encryptedName: toBase64(encCN1),
+        nameDecryptionNonce: toBase64(cn1N),
+        type: "album",
+        updationTime: 1700000000000000,
+        pubMagicMetadata: {
+            version: 1,
+            count: 1,
+            data: toBase64(encCollPM),
+            header: toBase64(collPMHeader),
+        },
+    };
+
+    // Collection 2: "Work" with no magic metadata
+    const ck2 = sodium.crypto_secretbox_keygen();
+    const { ciphertext: encCK2, nonce: ck2N } = encryptSecretbox(
+        ck2,
+        masterKey,
+    );
+    const { ciphertext: encCN2, nonce: cn2N } = encryptSecretbox(
+        new TextEncoder().encode("Work"),
+        ck2,
+    );
+    const rawColl2 = {
+        id: 20,
+        owner: { id: 42 },
+        encryptedKey: toBase64(encCK2),
+        keyDecryptionNonce: toBase64(ck2N),
+        encryptedName: toBase64(encCN2),
+        nameDecryptionNonce: toBase64(cn2N),
+        type: "folder",
+        updationTime: 1700000000000000,
+    };
+
+    // File 100 in coll 10: has metadata + pubMagicMetadata
+    const fk1 = sodium.crypto_secretstream_xchacha20poly1305_keygen();
+    const { ciphertext: encFK1, nonce: fk1N } = encryptSecretbox(fk1, ck1);
+    const meta1 = JSON.stringify({
+        title: "beach.jpg",
+        fileType: 0,
+        creationTime: 1700000000000000,
+        modificationTime: 1700000000000000,
+        latitude: 35.6762,
+        longitude: 139.6503,
+    });
+    const { ciphertext: encMeta1, header: meta1Header } = encryptStreamBlob(
+        new TextEncoder().encode(meta1),
+        fk1,
+    );
+    const pubMagic1 = JSON.stringify({
+        w: 3000,
+        h: 2000,
+        cameraMake: "SONY",
+        cameraModel: "DSC-RX1RM3",
+    });
+    const { ciphertext: encPM1, header: pm1Header } = encryptStreamBlob(
+        new TextEncoder().encode(pubMagic1),
+        fk1,
+    );
+
+    const rawFile1 = {
+        id: 100,
+        collectionID: 10,
+        ownerID: 42,
+        encryptedKey: toBase64(encFK1),
+        keyDecryptionNonce: toBase64(fk1N),
+        metadata: {
+            encryptedData: toBase64(encMeta1),
+            decryptionHeader: toBase64(meta1Header),
+        },
+        pubMagicMetadata: {
+            version: 1,
+            count: 1,
+            data: toBase64(encPM1),
+            header: toBase64(pm1Header),
+        },
+        file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
+        thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
+        updationTime: 1700000000000000,
+    };
+
+    // File 200 in coll 20: metadata only, no magic metadata
+    const fk2 = sodium.crypto_secretstream_xchacha20poly1305_keygen();
+    const { ciphertext: encFK2, nonce: fk2N } = encryptSecretbox(fk2, ck2);
+    const meta2 = JSON.stringify({
+        title: "diagram.png",
+        fileType: 0,
+        creationTime: 1710000000000000,
+        modificationTime: 1710000000000000,
+    });
+    const { ciphertext: encMeta2, header: meta2Header } = encryptStreamBlob(
+        new TextEncoder().encode(meta2),
+        fk2,
+    );
+
+    const rawFile2 = {
+        id: 200,
+        collectionID: 20,
+        ownerID: 42,
+        encryptedKey: toBase64(encFK2),
+        keyDecryptionNonce: toBase64(fk2N),
+        metadata: {
+            encryptedData: toBase64(encMeta2),
+            decryptionHeader: toBase64(meta2Header),
+        },
+        file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
+        thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
+        updationTime: 1710000000000000,
+    };
+
+    return {
+        verifier,
+        srpAttributes: {
+            srpUserID,
+            srpSalt: toBase64(srpSalt),
+            memLimit: TEST_MEM,
+            opsLimit: TEST_OPS,
+            kekSalt: toBase64(kekSalt),
+            isEmailMFAEnabled: false,
+        },
+        keyAttributes,
+        encryptedToken: toBase64(encToken),
+        collections: [rawColl1, rawColl2],
+        filesByCollection: { 10: [rawFile1], 20: [rawFile2] },
+    };
+};
+
+const buildMetaFetch = (m: MetaMockState) => {
+    let srpServer: SrpServer;
+    return (async (
+        input: RequestInfo | URL,
+        init?: RequestInit,
+    ): Promise<Response> => {
+        const url =
+            typeof input === "string"
+                ? input
+                : input instanceof URL
+                  ? input.href
+                  : input.url;
+        const path = new URL(url).pathname;
+        const json = (body: unknown) =>
+            new Response(JSON.stringify(body), {
+                status: 200,
+                headers: { "content-type": "application/json" },
+            });
+
+        if (path === "/users/srp/attributes")
+            return json({ attributes: m.srpAttributes });
+        if (path === "/users/srp/create-session") {
+            const body = JSON.parse(init?.body as string);
+            const serverKey = await SRP.genKey();
+            srpServer = new SrpServer(
+                SRP.params["4096"],
+                m.verifier,
+                serverKey,
+            );
+            const B = srpServer.computeB();
+            srpServer.setA(Buffer.from(body.srpA, "base64"));
+            return json({ sessionID: "s1", srpB: B.toString("base64") });
+        }
+        if (path === "/users/srp/verify-session") {
+            const body = JSON.parse(init?.body as string);
+            srpServer.checkM1(Buffer.from(body.srpM1, "base64"));
+            return json({
+                srpM2: srpServer.computeM2().toString("base64"),
+                id: 42,
+                keyAttributes: m.keyAttributes,
+                encryptedToken: m.encryptedToken,
+            });
+        }
+        if (path === "/collections/v2")
+            return json({ collections: m.collections });
+        if (path === "/collections/v2/diff") {
+            const collID = Number(
+                new URL(url).searchParams.get("collectionID"),
+            );
+            return json({
+                diff: m.filesByCollection[collID] ?? [],
+                hasMore: false,
+            });
+        }
+        return new Response("not found", { status: 404 });
+    }) as typeof globalThis.fetch;
+};
+
+beforeAll(async () => {
+    await init();
+    await sodium.ready;
+    mock = await buildMetaMock();
+    testDir = mkdtempSync(join(tmpdir(), "quak-meta-backup-test-"));
+});
+
+afterAll(() => {
+    if (testDir && existsSync(testDir))
+        rmSync(testDir, { recursive: true, force: true });
+});
+
+describe("quak backup-metadata", () => {
+    it("writes account.json with email and userID", async () => {
+        const outDir = join(testDir, "full");
+        const client = await Client.login({
+            email: TEST_EMAIL,
+            password: TEST_PASSWORD,
+            apiOptions: { fetch: buildMetaFetch(mock) },
+        });
+
+        await runMetadataBackup(client, outDir);
+
+        const account = JSON.parse(
+            readFileSync(join(outDir, "account.json"), "utf-8"),
+        );
+        expect(account.email).toBe(TEST_EMAIL);
+        expect(account.userID).toBe(42);
+    });
+
+    it("creates per-collection directories with _collection.json", async () => {
+        const outDir = join(testDir, "collections");
+        const client = await Client.login({
+            email: TEST_EMAIL,
+            password: TEST_PASSWORD,
+            apiOptions: { fetch: buildMetaFetch(mock) },
+        });
+
+        await runMetadataBackup(client, outDir);
+
+        const collDirs = readdirSync(join(outDir, "collections"));
+        expect(collDirs.length).toBe(2);
+
+        // Find the Vacation collection dir (prefixed with ID)
+        const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
+        expect(vacDir).toBeDefined();
+        const collMeta = JSON.parse(
+            readFileSync(
+                join(outDir, "collections", vacDir, "_collection.json"),
+                "utf-8",
+            ),
+        );
+        expect(collMeta.id).toBe(10);
+        expect(collMeta.name).toBe("Vacation");
+        expect(collMeta.type).toBe("album");
+    });
+
+    it("decrypts collection-level pubMagicMetadata", async () => {
+        const outDir = join(testDir, "coll-magic");
+        const client = await Client.login({
+            email: TEST_EMAIL,
+            password: TEST_PASSWORD,
+            apiOptions: { fetch: buildMetaFetch(mock) },
+        });
+
+        await runMetadataBackup(client, outDir);
+
+        const collDirs = readdirSync(join(outDir, "collections"));
+        const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
+        const collMeta = JSON.parse(
+            readFileSync(
+                join(outDir, "collections", vacDir, "_collection.json"),
+                "utf-8",
+            ),
+        );
+        expect(collMeta.pubMagicMetadata).toBeDefined();
+        expect(collMeta.pubMagicMetadata.coverID).toBe(999);
+        expect(collMeta.pubMagicMetadata.sortBy).toBe("date");
+    });
+
+    it("writes per-file JSON with all three metadata layers", async () => {
+        const outDir = join(testDir, "file-meta");
+        const client = await Client.login({
+            email: TEST_EMAIL,
+            password: TEST_PASSWORD,
+            apiOptions: { fetch: buildMetaFetch(mock) },
+        });
+
+        await runMetadataBackup(client, outDir);
+
+        const collDirs = readdirSync(join(outDir, "collections"));
+        const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
+        const fileMeta = JSON.parse(
+            readFileSync(
+                join(outDir, "collections", vacDir, "100.json"),
+                "utf-8",
+            ),
+        );
+        expect(fileMeta.id).toBe(100);
+        expect(fileMeta.metadata.title).toBe("beach.jpg");
+        expect(fileMeta.metadata.latitude).toBeCloseTo(35.6762);
+        expect(fileMeta.pubMagicMetadata.cameraMake).toBe("SONY");
+        expect(fileMeta.pubMagicMetadata.w).toBe(3000);
+    });
+
+    it("handles files with no magic metadata gracefully", async () => {
+        const outDir = join(testDir, "no-magic");
+        const client = await Client.login({
+            email: TEST_EMAIL,
+            password: TEST_PASSWORD,
+            apiOptions: { fetch: buildMetaFetch(mock) },
+        });
+
+        await runMetadataBackup(client, outDir);
+
+        const collDirs = readdirSync(join(outDir, "collections"));
+        const workDir = collDirs.find((d) => d.includes("Work"))!;
+        const fileMeta = JSON.parse(
+            readFileSync(
+                join(outDir, "collections", workDir, "200.json"),
+                "utf-8",
+            ),
+        );
+        expect(fileMeta.id).toBe(200);
+        expect(fileMeta.metadata.title).toBe("diagram.png");
+        expect(fileMeta.pubMagicMetadata).toBeUndefined();
+        expect(fileMeta.magicMetadata).toBeUndefined();
+    });
+
+    it("is incremental: second run does not fail", async () => {
+        const outDir = join(testDir, "incremental");
+        const client = await Client.login({
+            email: TEST_EMAIL,
+            password: TEST_PASSWORD,
+            apiOptions: { fetch: buildMetaFetch(mock) },
+        });
+
+        await runMetadataBackup(client, outDir);
+        await runMetadataBackup(client, outDir);
+
+        const account = JSON.parse(
+            readFileSync(join(outDir, "account.json"), "utf-8"),
+        );
+        expect(account.email).toBe(TEST_EMAIL);
+    });
+});