From f3958e911d742d5989d31ca2287284ae2b0bbc23 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 9 Jun 2026 12:41:34 -0400 Subject: [PATCH] Add quak backup-metadata: dump all decrypted metadata to plain JSON New command: quak backup-metadata Dumps every piece of decrypted account metadata into a directory tree of plain JSON files without downloading any file content. Layout: / account.json { email, userID } collections/ -/ _collection.json { id, name, type, pubMagicMetadata?, ... } .json { id, metadata, magicMetadata?, pubMagicMetadata? } Also adds collection-level magic metadata decryption (magicMetadata, pubMagicMetadata, sharedMagicMetadata) to decryptCollection, which was previously only done for files. The server sends these for visibility settings, sort order, cover photo selection, etc. 6 new tests covering: account.json, per-collection dirs with _collection.json, collection pubMagicMetadata decryption, per-file JSON with all three metadata layers, graceful handling of files with no magic metadata, and incremental re-run safety. 116 total. --- bin/quak.ts | 13 + src/metadata-backup.ts | 77 +++++ src/model/decrypt.ts | 3 + src/model/types.ts | 6 + test/cli/metadata-backup.test.ts | 468 +++++++++++++++++++++++++++++++ 5 files changed, 567 insertions(+) create mode 100644 src/metadata-backup.ts create mode 100644 test/cli/metadata-backup.test.ts diff --git a/bin/quak.ts b/bin/quak.ts index dd3c087..f5fcee1 100644 --- a/bin/quak.ts +++ b/bin/quak.ts @@ -9,6 +9,7 @@ import envPaths from "env-paths"; import { Client, type ClientSnapshot } from "../src/client.js"; import { init } from "../src/crypto/index.js"; import { runBackup } from "../src/backup.js"; +import { runMetadataBackup } from "../src/metadata-backup.js"; import { listMissingThumbnails, fixMissingThumbnails, @@ -333,6 +334,18 @@ program }, ); +program + .command("backup-metadata") + .description( + "Dump all decrypted account metadata (no file content) to a directory", + ) + .argument("", "Output directory") + .action(async (dir: string) => { + await init(); + const client = requireSession(); + await runMetadataBackup(client, dir, (msg) => stderr.write(msg + "\n")); + }); + program .command("backup") .description( diff --git a/src/metadata-backup.ts b/src/metadata-backup.ts new file mode 100644 index 0000000..2b622ca --- /dev/null +++ b/src/metadata-backup.ts @@ -0,0 +1,77 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { Client } from "./client.js"; + +export type ProgressCallback = (message: string) => void; + +const sanitizePath = (name: string): string => + name.replace(/[/\\:*?"<>|]/g, "_").replace(/^\.+/, "_"); + +export const runMetadataBackup = async ( + client: Client, + outDir: string, + onProgress?: ProgressCallback, +): Promise => { + const log = onProgress ?? (() => {}); + + mkdirSync(outDir, { recursive: true }); + mkdirSync(join(outDir, "collections"), { recursive: true }); + + const { email, userID } = client.whoami(); + writeFileSync( + join(outDir, "account.json"), + JSON.stringify({ email, userID }, null, 2), + ); + + log("Fetching collections..."); + const collections = await client.listCollections(); + + for (const col of collections) { + const dirName = `${col.id}-${sanitizePath(col.name || "unnamed")}`; + const colDir = join(outDir, "collections", dirName); + mkdirSync(colDir, { recursive: true }); + + const collectionMeta: Record = { + id: col.id, + name: col.name, + type: col.type, + ownerID: col.ownerID, + isShared: col.isShared, + updationTime: col.updationTime, + }; + if (col.magicMetadata) collectionMeta.magicMetadata = col.magicMetadata; + if (col.pubMagicMetadata) + collectionMeta.pubMagicMetadata = col.pubMagicMetadata; + if (col.sharedMagicMetadata) + collectionMeta.sharedMagicMetadata = col.sharedMagicMetadata; + + writeFileSync( + join(colDir, "_collection.json"), + JSON.stringify(collectionMeta, null, 2), + ); + + log(`[${col.name}] Fetching files...`); + const files = await client.listFiles(col.id, col.key); + log(`[${col.name}] ${files.length} file(s)`); + + for (const file of files) { + const fileMeta: Record = { + id: file.id, + collectionID: file.collectionID, + ownerID: file.ownerID, + metadata: file.metadata, + updationTime: file.updationTime, + }; + if (file.magicMetadata) fileMeta.magicMetadata = file.magicMetadata; + if (file.pubMagicMetadata) + fileMeta.pubMagicMetadata = file.pubMagicMetadata; + + writeFileSync( + join(colDir, `${file.id}.json`), + JSON.stringify(fileMeta, null, 2), + ); + } + } + + log("Metadata backup complete."); +}; diff --git a/src/model/decrypt.ts b/src/model/decrypt.ts index 5154357..4851963 100644 --- a/src/model/decrypt.ts +++ b/src/model/decrypt.ts @@ -57,6 +57,9 @@ export const decryptCollection = ( type: parseCollectionType(raw.type), updationTime: raw.updationTime, isShared: currentUserID !== undefined && raw.owner.id !== currentUserID, + magicMetadata: decryptMagicMetadata(raw.magicMetadata, key), + pubMagicMetadata: decryptMagicMetadata(raw.pubMagicMetadata, key), + sharedMagicMetadata: decryptMagicMetadata(raw.sharedMagicMetadata, key), }; }; diff --git a/src/model/types.ts b/src/model/types.ts index 42cb5b5..5aee6ab 100644 --- a/src/model/types.ts +++ b/src/model/types.ts @@ -15,6 +15,9 @@ export interface Collection { type: CollectionType; updationTime: Microseconds; isShared: boolean; + magicMetadata?: Record; + pubMagicMetadata?: Record; + sharedMagicMetadata?: Record; } export type FileType = "image" | "video" | "livePhoto" | "unknown"; @@ -59,6 +62,9 @@ export interface RawCollection { type: string; updationTime: number; isDeleted?: boolean; + magicMetadata?: RawMagicMetadata; + pubMagicMetadata?: RawMagicMetadata; + sharedMagicMetadata?: RawMagicMetadata; } export interface RawMagicMetadata { diff --git a/test/cli/metadata-backup.test.ts b/test/cli/metadata-backup.test.ts new file mode 100644 index 0000000..0d86b3e --- /dev/null +++ b/test/cli/metadata-backup.test.ts @@ -0,0 +1,468 @@ +/** + * Tests for `quak backup-metadata `. + * + * This command dumps all decrypted account metadata into a directory + * tree of plain JSON files, without downloading any file content. It + * is fast (no multi-megabyte downloads) and produces a complete + * plaintext record of every collection name, file title, creation + * date, GPS coordinate, camera model, caption, face label, and any + * other metadata the Ente clients have attached. + * + * Layout: + * + * / + * account.json { email, userID } + * collections/ + * -/ + * _collection.json { id, name, type, magicMetadata?, ... } + * .json { id, metadata, magicMetadata?, pubMagicMetadata? } + * + * The test builds a mock server with two collections, each with files + * that have different combinations of metadata layers, and verifies + * the output tree is correct and complete. + */ + +import { + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, +} from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import sodium from "libsodium-wrappers-sumo"; +import { SRP, SrpServer } from "fast-srp-hap"; +import { beforeAll, afterAll, describe, expect, it } from "vitest"; +import { + init, + toBase64, + deriveKEK, + deriveLoginSubkey, +} from "../../src/crypto/index.js"; +import { Client } from "../../src/client.js"; +import { runMetadataBackup } from "../../src/metadata-backup.js"; +import type { KeyAttributes } from "../../src/auth/types.js"; + +const TEST_EMAIL = "metabackup@example.com"; +const TEST_PASSWORD = "metapass"; +const TEST_OPS = 2; +const TEST_MEM = 64 * 1024 * 1024; + +interface MetaMockState { + verifier: Buffer; + srpAttributes: Record; + keyAttributes: KeyAttributes; + encryptedToken: string; + collections: Record[]; + filesByCollection: Record[]>; +} + +let mock: MetaMockState; +let testDir: string; + +const encryptSecretbox = (plaintext: Uint8Array, key: Uint8Array) => { + const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + const ciphertext = sodium.crypto_secretbox_easy(plaintext, nonce, key); + return { ciphertext, nonce }; +}; + +const encryptStreamBlob = (plaintext: Uint8Array, key: Uint8Array) => { + const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key); + const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push( + push.state, + plaintext, + null, + sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, + ); + return { ciphertext, header: push.header }; +}; + +const buildMetaMock = async (): Promise => { + const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES); + const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM); + const loginSubKeyBytes = deriveLoginSubkey(kek); + const srpUserID = "meta-srp"; + const srpSalt = sodium.randombytes_buf(16); + const verifier = SRP.computeVerifier( + SRP.params["4096"], + Buffer.from(srpSalt), + Buffer.from(srpUserID), + Buffer.from(loginSubKeyBytes), + ); + + const masterKey = sodium.randombytes_buf(32); + const { ciphertext: encMK, nonce: mkNonce } = encryptSecretbox( + masterKey, + kek, + ); + const kp = sodium.crypto_box_keypair(); + const { ciphertext: encSK, nonce: skNonce } = encryptSecretbox( + kp.privateKey, + masterKey, + ); + const tokenBytes = sodium.randombytes_buf(32); + const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey); + + const keyAttributes: KeyAttributes = { + kekSalt: toBase64(kekSalt), + encryptedKey: toBase64(encMK), + keyDecryptionNonce: toBase64(mkNonce), + publicKey: toBase64(kp.publicKey), + encryptedSecretKey: toBase64(encSK), + secretKeyDecryptionNonce: toBase64(skNonce), + memLimit: TEST_MEM, + opsLimit: TEST_OPS, + }; + + // Collection 1: "Vacation" with collection-level pubMagicMetadata + const ck1 = sodium.crypto_secretbox_keygen(); + const { ciphertext: encCK1, nonce: ck1N } = encryptSecretbox( + ck1, + masterKey, + ); + const { ciphertext: encCN1, nonce: cn1N } = encryptSecretbox( + new TextEncoder().encode("Vacation"), + ck1, + ); + const collPubMagic = JSON.stringify({ coverID: 999, sortBy: "date" }); + const { ciphertext: encCollPM, header: collPMHeader } = encryptStreamBlob( + new TextEncoder().encode(collPubMagic), + ck1, + ); + + const rawColl1 = { + id: 10, + owner: { id: 42 }, + encryptedKey: toBase64(encCK1), + keyDecryptionNonce: toBase64(ck1N), + encryptedName: toBase64(encCN1), + nameDecryptionNonce: toBase64(cn1N), + type: "album", + updationTime: 1700000000000000, + pubMagicMetadata: { + version: 1, + count: 1, + data: toBase64(encCollPM), + header: toBase64(collPMHeader), + }, + }; + + // Collection 2: "Work" with no magic metadata + const ck2 = sodium.crypto_secretbox_keygen(); + const { ciphertext: encCK2, nonce: ck2N } = encryptSecretbox( + ck2, + masterKey, + ); + const { ciphertext: encCN2, nonce: cn2N } = encryptSecretbox( + new TextEncoder().encode("Work"), + ck2, + ); + const rawColl2 = { + id: 20, + owner: { id: 42 }, + encryptedKey: toBase64(encCK2), + keyDecryptionNonce: toBase64(ck2N), + encryptedName: toBase64(encCN2), + nameDecryptionNonce: toBase64(cn2N), + type: "folder", + updationTime: 1700000000000000, + }; + + // File 100 in coll 10: has metadata + pubMagicMetadata + const fk1 = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { ciphertext: encFK1, nonce: fk1N } = encryptSecretbox(fk1, ck1); + const meta1 = JSON.stringify({ + title: "beach.jpg", + fileType: 0, + creationTime: 1700000000000000, + modificationTime: 1700000000000000, + latitude: 35.6762, + longitude: 139.6503, + }); + const { ciphertext: encMeta1, header: meta1Header } = encryptStreamBlob( + new TextEncoder().encode(meta1), + fk1, + ); + const pubMagic1 = JSON.stringify({ + w: 3000, + h: 2000, + cameraMake: "SONY", + cameraModel: "DSC-RX1RM3", + }); + const { ciphertext: encPM1, header: pm1Header } = encryptStreamBlob( + new TextEncoder().encode(pubMagic1), + fk1, + ); + + const rawFile1 = { + id: 100, + collectionID: 10, + ownerID: 42, + encryptedKey: toBase64(encFK1), + keyDecryptionNonce: toBase64(fk1N), + metadata: { + encryptedData: toBase64(encMeta1), + decryptionHeader: toBase64(meta1Header), + }, + pubMagicMetadata: { + version: 1, + count: 1, + data: toBase64(encPM1), + header: toBase64(pm1Header), + }, + file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, + thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, + updationTime: 1700000000000000, + }; + + // File 200 in coll 20: metadata only, no magic metadata + const fk2 = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { ciphertext: encFK2, nonce: fk2N } = encryptSecretbox(fk2, ck2); + const meta2 = JSON.stringify({ + title: "diagram.png", + fileType: 0, + creationTime: 1710000000000000, + modificationTime: 1710000000000000, + }); + const { ciphertext: encMeta2, header: meta2Header } = encryptStreamBlob( + new TextEncoder().encode(meta2), + fk2, + ); + + const rawFile2 = { + id: 200, + collectionID: 20, + ownerID: 42, + encryptedKey: toBase64(encFK2), + keyDecryptionNonce: toBase64(fk2N), + metadata: { + encryptedData: toBase64(encMeta2), + decryptionHeader: toBase64(meta2Header), + }, + file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, + thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, + updationTime: 1710000000000000, + }; + + return { + verifier, + srpAttributes: { + srpUserID, + srpSalt: toBase64(srpSalt), + memLimit: TEST_MEM, + opsLimit: TEST_OPS, + kekSalt: toBase64(kekSalt), + isEmailMFAEnabled: false, + }, + keyAttributes, + encryptedToken: toBase64(encToken), + collections: [rawColl1, rawColl2], + filesByCollection: { 10: [rawFile1], 20: [rawFile2] }, + }; +}; + +const buildMetaFetch = (m: MetaMockState) => { + let srpServer: SrpServer; + return (async ( + input: RequestInfo | URL, + init?: RequestInit, + ): Promise => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.href + : input.url; + const path = new URL(url).pathname; + const json = (body: unknown) => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); + + if (path === "/users/srp/attributes") + return json({ attributes: m.srpAttributes }); + if (path === "/users/srp/create-session") { + const body = JSON.parse(init?.body as string); + const serverKey = await SRP.genKey(); + srpServer = new SrpServer( + SRP.params["4096"], + m.verifier, + serverKey, + ); + const B = srpServer.computeB(); + srpServer.setA(Buffer.from(body.srpA, "base64")); + return json({ sessionID: "s1", srpB: B.toString("base64") }); + } + if (path === "/users/srp/verify-session") { + const body = JSON.parse(init?.body as string); + srpServer.checkM1(Buffer.from(body.srpM1, "base64")); + return json({ + srpM2: srpServer.computeM2().toString("base64"), + id: 42, + keyAttributes: m.keyAttributes, + encryptedToken: m.encryptedToken, + }); + } + if (path === "/collections/v2") + return json({ collections: m.collections }); + if (path === "/collections/v2/diff") { + const collID = Number( + new URL(url).searchParams.get("collectionID"), + ); + return json({ + diff: m.filesByCollection[collID] ?? [], + hasMore: false, + }); + } + return new Response("not found", { status: 404 }); + }) as typeof globalThis.fetch; +}; + +beforeAll(async () => { + await init(); + await sodium.ready; + mock = await buildMetaMock(); + testDir = mkdtempSync(join(tmpdir(), "quak-meta-backup-test-")); +}); + +afterAll(() => { + if (testDir && existsSync(testDir)) + rmSync(testDir, { recursive: true, force: true }); +}); + +describe("quak backup-metadata", () => { + it("writes account.json with email and userID", async () => { + const outDir = join(testDir, "full"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMetaFetch(mock) }, + }); + + await runMetadataBackup(client, outDir); + + const account = JSON.parse( + readFileSync(join(outDir, "account.json"), "utf-8"), + ); + expect(account.email).toBe(TEST_EMAIL); + expect(account.userID).toBe(42); + }); + + it("creates per-collection directories with _collection.json", async () => { + const outDir = join(testDir, "collections"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMetaFetch(mock) }, + }); + + await runMetadataBackup(client, outDir); + + const collDirs = readdirSync(join(outDir, "collections")); + expect(collDirs.length).toBe(2); + + // Find the Vacation collection dir (prefixed with ID) + const vacDir = collDirs.find((d) => d.includes("Vacation"))!; + expect(vacDir).toBeDefined(); + const collMeta = JSON.parse( + readFileSync( + join(outDir, "collections", vacDir, "_collection.json"), + "utf-8", + ), + ); + expect(collMeta.id).toBe(10); + expect(collMeta.name).toBe("Vacation"); + expect(collMeta.type).toBe("album"); + }); + + it("decrypts collection-level pubMagicMetadata", async () => { + const outDir = join(testDir, "coll-magic"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMetaFetch(mock) }, + }); + + await runMetadataBackup(client, outDir); + + const collDirs = readdirSync(join(outDir, "collections")); + const vacDir = collDirs.find((d) => d.includes("Vacation"))!; + const collMeta = JSON.parse( + readFileSync( + join(outDir, "collections", vacDir, "_collection.json"), + "utf-8", + ), + ); + expect(collMeta.pubMagicMetadata).toBeDefined(); + expect(collMeta.pubMagicMetadata.coverID).toBe(999); + expect(collMeta.pubMagicMetadata.sortBy).toBe("date"); + }); + + it("writes per-file JSON with all three metadata layers", async () => { + const outDir = join(testDir, "file-meta"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMetaFetch(mock) }, + }); + + await runMetadataBackup(client, outDir); + + const collDirs = readdirSync(join(outDir, "collections")); + const vacDir = collDirs.find((d) => d.includes("Vacation"))!; + const fileMeta = JSON.parse( + readFileSync( + join(outDir, "collections", vacDir, "100.json"), + "utf-8", + ), + ); + expect(fileMeta.id).toBe(100); + expect(fileMeta.metadata.title).toBe("beach.jpg"); + expect(fileMeta.metadata.latitude).toBeCloseTo(35.6762); + expect(fileMeta.pubMagicMetadata.cameraMake).toBe("SONY"); + expect(fileMeta.pubMagicMetadata.w).toBe(3000); + }); + + it("handles files with no magic metadata gracefully", async () => { + const outDir = join(testDir, "no-magic"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMetaFetch(mock) }, + }); + + await runMetadataBackup(client, outDir); + + const collDirs = readdirSync(join(outDir, "collections")); + const workDir = collDirs.find((d) => d.includes("Work"))!; + const fileMeta = JSON.parse( + readFileSync( + join(outDir, "collections", workDir, "200.json"), + "utf-8", + ), + ); + expect(fileMeta.id).toBe(200); + expect(fileMeta.metadata.title).toBe("diagram.png"); + expect(fileMeta.pubMagicMetadata).toBeUndefined(); + expect(fileMeta.magicMetadata).toBeUndefined(); + }); + + it("is incremental: second run does not fail", async () => { + const outDir = join(testDir, "incremental"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMetaFetch(mock) }, + }); + + await runMetadataBackup(client, outDir); + await runMetadataBackup(client, outDir); + + const account = JSON.parse( + readFileSync(join(outDir, "account.json"), "utf-8"), + ); + expect(account.email).toBe(TEST_EMAIL); + }); +});