Merge: quak backup-metadata command

Dumps all decrypted account metadata to a directory tree of plain JSON
files. No file content downloads. Includes collection-level magic
metadata decryption (visibility, sort order, cover photo) which was
previously missing. 6 new tests, 116 total.
This commit is contained in:
2026-06-09 12:41:43 -04:00
5 changed files with 567 additions and 0 deletions

View File

@@ -9,6 +9,7 @@ import envPaths from "env-paths";
import { Client, type ClientSnapshot } from "../src/client.js";
import { init } from "../src/crypto/index.js";
import { runBackup } from "../src/backup.js";
import { runMetadataBackup } from "../src/metadata-backup.js";
import {
listMissingThumbnails,
fixMissingThumbnails,
@@ -333,6 +334,18 @@ program
},
);
program
.command("backup-metadata")
.description(
"Dump all decrypted account metadata (no file content) to a directory",
)
.argument("<dir>", "Output directory")
.action(async (dir: string) => {
await init();
const client = requireSession();
await runMetadataBackup(client, dir, (msg) => stderr.write(msg + "\n"));
});
program
.command("backup")
.description(

77
src/metadata-backup.ts Normal file
View File

@@ -0,0 +1,77 @@
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import type { Client } from "./client.js";
export type ProgressCallback = (message: string) => void;
const sanitizePath = (name: string): string =>
name.replace(/[/\\:*?"<>|]/g, "_").replace(/^\.+/, "_");
export const runMetadataBackup = async (
client: Client,
outDir: string,
onProgress?: ProgressCallback,
): Promise<void> => {
const log = onProgress ?? (() => {});
mkdirSync(outDir, { recursive: true });
mkdirSync(join(outDir, "collections"), { recursive: true });
const { email, userID } = client.whoami();
writeFileSync(
join(outDir, "account.json"),
JSON.stringify({ email, userID }, null, 2),
);
log("Fetching collections...");
const collections = await client.listCollections();
for (const col of collections) {
const dirName = `${col.id}-${sanitizePath(col.name || "unnamed")}`;
const colDir = join(outDir, "collections", dirName);
mkdirSync(colDir, { recursive: true });
const collectionMeta: Record<string, unknown> = {
id: col.id,
name: col.name,
type: col.type,
ownerID: col.ownerID,
isShared: col.isShared,
updationTime: col.updationTime,
};
if (col.magicMetadata) collectionMeta.magicMetadata = col.magicMetadata;
if (col.pubMagicMetadata)
collectionMeta.pubMagicMetadata = col.pubMagicMetadata;
if (col.sharedMagicMetadata)
collectionMeta.sharedMagicMetadata = col.sharedMagicMetadata;
writeFileSync(
join(colDir, "_collection.json"),
JSON.stringify(collectionMeta, null, 2),
);
log(`[${col.name}] Fetching files...`);
const files = await client.listFiles(col.id, col.key);
log(`[${col.name}] ${files.length} file(s)`);
for (const file of files) {
const fileMeta: Record<string, unknown> = {
id: file.id,
collectionID: file.collectionID,
ownerID: file.ownerID,
metadata: file.metadata,
updationTime: file.updationTime,
};
if (file.magicMetadata) fileMeta.magicMetadata = file.magicMetadata;
if (file.pubMagicMetadata)
fileMeta.pubMagicMetadata = file.pubMagicMetadata;
writeFileSync(
join(colDir, `${file.id}.json`),
JSON.stringify(fileMeta, null, 2),
);
}
}
log("Metadata backup complete.");
};

View File

@@ -57,6 +57,9 @@ export const decryptCollection = (
type: parseCollectionType(raw.type),
updationTime: raw.updationTime,
isShared: currentUserID !== undefined && raw.owner.id !== currentUserID,
magicMetadata: decryptMagicMetadata(raw.magicMetadata, key),
pubMagicMetadata: decryptMagicMetadata(raw.pubMagicMetadata, key),
sharedMagicMetadata: decryptMagicMetadata(raw.sharedMagicMetadata, key),
};
};

View File

@@ -15,6 +15,9 @@ export interface Collection {
type: CollectionType;
updationTime: Microseconds;
isShared: boolean;
magicMetadata?: Record<string, unknown>;
pubMagicMetadata?: Record<string, unknown>;
sharedMagicMetadata?: Record<string, unknown>;
}
export type FileType = "image" | "video" | "livePhoto" | "unknown";
@@ -59,6 +62,9 @@ export interface RawCollection {
type: string;
updationTime: number;
isDeleted?: boolean;
magicMetadata?: RawMagicMetadata;
pubMagicMetadata?: RawMagicMetadata;
sharedMagicMetadata?: RawMagicMetadata;
}
export interface RawMagicMetadata {

View File

@@ -0,0 +1,468 @@
/**
* Tests for `quak backup-metadata <dir>`.
*
* This command dumps all decrypted account metadata into a directory
* tree of plain JSON files, without downloading any file content. It
* is fast (no multi-megabyte downloads) and produces a complete
* plaintext record of every collection name, file title, creation
* date, GPS coordinate, camera model, caption, face label, and any
* other metadata the Ente clients have attached.
*
* Layout:
*
* <dir>/
* account.json { email, userID }
* collections/
* <id>-<sanitized-name>/
* _collection.json { id, name, type, magicMetadata?, ... }
* <fileID>.json { id, metadata, magicMetadata?, pubMagicMetadata? }
*
* The test builds a mock server with two collections, each with files
* that have different combinations of metadata layers, and verifies
* the output tree is correct and complete.
*/
import {
existsSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
} from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import sodium from "libsodium-wrappers-sumo";
import { SRP, SrpServer } from "fast-srp-hap";
import { beforeAll, afterAll, describe, expect, it } from "vitest";
import {
init,
toBase64,
deriveKEK,
deriveLoginSubkey,
} from "../../src/crypto/index.js";
import { Client } from "../../src/client.js";
import { runMetadataBackup } from "../../src/metadata-backup.js";
import type { KeyAttributes } from "../../src/auth/types.js";
const TEST_EMAIL = "metabackup@example.com";
const TEST_PASSWORD = "metapass";
const TEST_OPS = 2;
const TEST_MEM = 64 * 1024 * 1024;
interface MetaMockState {
verifier: Buffer;
srpAttributes: Record<string, unknown>;
keyAttributes: KeyAttributes;
encryptedToken: string;
collections: Record<string, unknown>[];
filesByCollection: Record<number, Record<string, unknown>[]>;
}
let mock: MetaMockState;
let testDir: string;
const encryptSecretbox = (plaintext: Uint8Array, key: Uint8Array) => {
const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
const ciphertext = sodium.crypto_secretbox_easy(plaintext, nonce, key);
return { ciphertext, nonce };
};
const encryptStreamBlob = (plaintext: Uint8Array, key: Uint8Array) => {
const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push(
push.state,
plaintext,
null,
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
);
return { ciphertext, header: push.header };
};
const buildMetaMock = async (): Promise<MetaMockState> => {
const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM);
const loginSubKeyBytes = deriveLoginSubkey(kek);
const srpUserID = "meta-srp";
const srpSalt = sodium.randombytes_buf(16);
const verifier = SRP.computeVerifier(
SRP.params["4096"],
Buffer.from(srpSalt),
Buffer.from(srpUserID),
Buffer.from(loginSubKeyBytes),
);
const masterKey = sodium.randombytes_buf(32);
const { ciphertext: encMK, nonce: mkNonce } = encryptSecretbox(
masterKey,
kek,
);
const kp = sodium.crypto_box_keypair();
const { ciphertext: encSK, nonce: skNonce } = encryptSecretbox(
kp.privateKey,
masterKey,
);
const tokenBytes = sodium.randombytes_buf(32);
const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey);
const keyAttributes: KeyAttributes = {
kekSalt: toBase64(kekSalt),
encryptedKey: toBase64(encMK),
keyDecryptionNonce: toBase64(mkNonce),
publicKey: toBase64(kp.publicKey),
encryptedSecretKey: toBase64(encSK),
secretKeyDecryptionNonce: toBase64(skNonce),
memLimit: TEST_MEM,
opsLimit: TEST_OPS,
};
// Collection 1: "Vacation" with collection-level pubMagicMetadata
const ck1 = sodium.crypto_secretbox_keygen();
const { ciphertext: encCK1, nonce: ck1N } = encryptSecretbox(
ck1,
masterKey,
);
const { ciphertext: encCN1, nonce: cn1N } = encryptSecretbox(
new TextEncoder().encode("Vacation"),
ck1,
);
const collPubMagic = JSON.stringify({ coverID: 999, sortBy: "date" });
const { ciphertext: encCollPM, header: collPMHeader } = encryptStreamBlob(
new TextEncoder().encode(collPubMagic),
ck1,
);
const rawColl1 = {
id: 10,
owner: { id: 42 },
encryptedKey: toBase64(encCK1),
keyDecryptionNonce: toBase64(ck1N),
encryptedName: toBase64(encCN1),
nameDecryptionNonce: toBase64(cn1N),
type: "album",
updationTime: 1700000000000000,
pubMagicMetadata: {
version: 1,
count: 1,
data: toBase64(encCollPM),
header: toBase64(collPMHeader),
},
};
// Collection 2: "Work" with no magic metadata
const ck2 = sodium.crypto_secretbox_keygen();
const { ciphertext: encCK2, nonce: ck2N } = encryptSecretbox(
ck2,
masterKey,
);
const { ciphertext: encCN2, nonce: cn2N } = encryptSecretbox(
new TextEncoder().encode("Work"),
ck2,
);
const rawColl2 = {
id: 20,
owner: { id: 42 },
encryptedKey: toBase64(encCK2),
keyDecryptionNonce: toBase64(ck2N),
encryptedName: toBase64(encCN2),
nameDecryptionNonce: toBase64(cn2N),
type: "folder",
updationTime: 1700000000000000,
};
// File 100 in coll 10: has metadata + pubMagicMetadata
const fk1 = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { ciphertext: encFK1, nonce: fk1N } = encryptSecretbox(fk1, ck1);
const meta1 = JSON.stringify({
title: "beach.jpg",
fileType: 0,
creationTime: 1700000000000000,
modificationTime: 1700000000000000,
latitude: 35.6762,
longitude: 139.6503,
});
const { ciphertext: encMeta1, header: meta1Header } = encryptStreamBlob(
new TextEncoder().encode(meta1),
fk1,
);
const pubMagic1 = JSON.stringify({
w: 3000,
h: 2000,
cameraMake: "SONY",
cameraModel: "DSC-RX1RM3",
});
const { ciphertext: encPM1, header: pm1Header } = encryptStreamBlob(
new TextEncoder().encode(pubMagic1),
fk1,
);
const rawFile1 = {
id: 100,
collectionID: 10,
ownerID: 42,
encryptedKey: toBase64(encFK1),
keyDecryptionNonce: toBase64(fk1N),
metadata: {
encryptedData: toBase64(encMeta1),
decryptionHeader: toBase64(meta1Header),
},
pubMagicMetadata: {
version: 1,
count: 1,
data: toBase64(encPM1),
header: toBase64(pm1Header),
},
file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
updationTime: 1700000000000000,
};
// File 200 in coll 20: metadata only, no magic metadata
const fk2 = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { ciphertext: encFK2, nonce: fk2N } = encryptSecretbox(fk2, ck2);
const meta2 = JSON.stringify({
title: "diagram.png",
fileType: 0,
creationTime: 1710000000000000,
modificationTime: 1710000000000000,
});
const { ciphertext: encMeta2, header: meta2Header } = encryptStreamBlob(
new TextEncoder().encode(meta2),
fk2,
);
const rawFile2 = {
id: 200,
collectionID: 20,
ownerID: 42,
encryptedKey: toBase64(encFK2),
keyDecryptionNonce: toBase64(fk2N),
metadata: {
encryptedData: toBase64(encMeta2),
decryptionHeader: toBase64(meta2Header),
},
file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
updationTime: 1710000000000000,
};
return {
verifier,
srpAttributes: {
srpUserID,
srpSalt: toBase64(srpSalt),
memLimit: TEST_MEM,
opsLimit: TEST_OPS,
kekSalt: toBase64(kekSalt),
isEmailMFAEnabled: false,
},
keyAttributes,
encryptedToken: toBase64(encToken),
collections: [rawColl1, rawColl2],
filesByCollection: { 10: [rawFile1], 20: [rawFile2] },
};
};
const buildMetaFetch = (m: MetaMockState) => {
let srpServer: SrpServer;
return (async (
input: RequestInfo | URL,
init?: RequestInit,
): Promise<Response> => {
const url =
typeof input === "string"
? input
: input instanceof URL
? input.href
: input.url;
const path = new URL(url).pathname;
const json = (body: unknown) =>
new Response(JSON.stringify(body), {
status: 200,
headers: { "content-type": "application/json" },
});
if (path === "/users/srp/attributes")
return json({ attributes: m.srpAttributes });
if (path === "/users/srp/create-session") {
const body = JSON.parse(init?.body as string);
const serverKey = await SRP.genKey();
srpServer = new SrpServer(
SRP.params["4096"],
m.verifier,
serverKey,
);
const B = srpServer.computeB();
srpServer.setA(Buffer.from(body.srpA, "base64"));
return json({ sessionID: "s1", srpB: B.toString("base64") });
}
if (path === "/users/srp/verify-session") {
const body = JSON.parse(init?.body as string);
srpServer.checkM1(Buffer.from(body.srpM1, "base64"));
return json({
srpM2: srpServer.computeM2().toString("base64"),
id: 42,
keyAttributes: m.keyAttributes,
encryptedToken: m.encryptedToken,
});
}
if (path === "/collections/v2")
return json({ collections: m.collections });
if (path === "/collections/v2/diff") {
const collID = Number(
new URL(url).searchParams.get("collectionID"),
);
return json({
diff: m.filesByCollection[collID] ?? [],
hasMore: false,
});
}
return new Response("not found", { status: 404 });
}) as typeof globalThis.fetch;
};
beforeAll(async () => {
await init();
await sodium.ready;
mock = await buildMetaMock();
testDir = mkdtempSync(join(tmpdir(), "quak-meta-backup-test-"));
});
afterAll(() => {
if (testDir && existsSync(testDir))
rmSync(testDir, { recursive: true, force: true });
});
describe("quak backup-metadata", () => {
it("writes account.json with email and userID", async () => {
const outDir = join(testDir, "full");
const client = await Client.login({
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const account = JSON.parse(
readFileSync(join(outDir, "account.json"), "utf-8"),
);
expect(account.email).toBe(TEST_EMAIL);
expect(account.userID).toBe(42);
});
it("creates per-collection directories with _collection.json", async () => {
const outDir = join(testDir, "collections");
const client = await Client.login({
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const collDirs = readdirSync(join(outDir, "collections"));
expect(collDirs.length).toBe(2);
// Find the Vacation collection dir (prefixed with ID)
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
expect(vacDir).toBeDefined();
const collMeta = JSON.parse(
readFileSync(
join(outDir, "collections", vacDir, "_collection.json"),
"utf-8",
),
);
expect(collMeta.id).toBe(10);
expect(collMeta.name).toBe("Vacation");
expect(collMeta.type).toBe("album");
});
it("decrypts collection-level pubMagicMetadata", async () => {
const outDir = join(testDir, "coll-magic");
const client = await Client.login({
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const collDirs = readdirSync(join(outDir, "collections"));
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
const collMeta = JSON.parse(
readFileSync(
join(outDir, "collections", vacDir, "_collection.json"),
"utf-8",
),
);
expect(collMeta.pubMagicMetadata).toBeDefined();
expect(collMeta.pubMagicMetadata.coverID).toBe(999);
expect(collMeta.pubMagicMetadata.sortBy).toBe("date");
});
it("writes per-file JSON with all three metadata layers", async () => {
const outDir = join(testDir, "file-meta");
const client = await Client.login({
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const collDirs = readdirSync(join(outDir, "collections"));
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
const fileMeta = JSON.parse(
readFileSync(
join(outDir, "collections", vacDir, "100.json"),
"utf-8",
),
);
expect(fileMeta.id).toBe(100);
expect(fileMeta.metadata.title).toBe("beach.jpg");
expect(fileMeta.metadata.latitude).toBeCloseTo(35.6762);
expect(fileMeta.pubMagicMetadata.cameraMake).toBe("SONY");
expect(fileMeta.pubMagicMetadata.w).toBe(3000);
});
it("handles files with no magic metadata gracefully", async () => {
const outDir = join(testDir, "no-magic");
const client = await Client.login({
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const collDirs = readdirSync(join(outDir, "collections"));
const workDir = collDirs.find((d) => d.includes("Work"))!;
const fileMeta = JSON.parse(
readFileSync(
join(outDir, "collections", workDir, "200.json"),
"utf-8",
),
);
expect(fileMeta.id).toBe(200);
expect(fileMeta.metadata.title).toBe("diagram.png");
expect(fileMeta.pubMagicMetadata).toBeUndefined();
expect(fileMeta.magicMetadata).toBeUndefined();
});
it("is incremental: second run does not fail", async () => {
const outDir = join(testDir, "incremental");
const client = await Client.login({
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
await runMetadataBackup(client, outDir);
const account = JSON.parse(
readFileSync(join(outDir, "account.json"), "utf-8"),
);
expect(account.email).toBe(TEST_EMAIL);
});
});