Add quak backup-metadata: dump all decrypted metadata to plain JSON
New command: quak backup-metadata <dir>
Dumps every piece of decrypted account metadata into a directory tree
of plain JSON files without downloading any file content. Layout:
<dir>/
account.json { email, userID }
collections/
<id>-<name>/
_collection.json { id, name, type, pubMagicMetadata?, ... }
<fileID>.json { id, metadata, magicMetadata?, pubMagicMetadata? }
Also adds collection-level magic metadata decryption (magicMetadata,
pubMagicMetadata, sharedMagicMetadata) to decryptCollection, which was
previously only done for files. The server sends these for visibility
settings, sort order, cover photo selection, etc.
6 new tests covering: account.json, per-collection dirs with
_collection.json, collection pubMagicMetadata decryption, per-file
JSON with all three metadata layers, graceful handling of files with
no magic metadata, and incremental re-run safety. 116 total.
This commit is contained in:
468
test/cli/metadata-backup.test.ts
Normal file
468
test/cli/metadata-backup.test.ts
Normal file
@@ -0,0 +1,468 @@
|
||||
/**
|
||||
* Tests for `quak backup-metadata <dir>`.
|
||||
*
|
||||
* This command dumps all decrypted account metadata into a directory
|
||||
* tree of plain JSON files, without downloading any file content. It
|
||||
* is fast (no multi-megabyte downloads) and produces a complete
|
||||
* plaintext record of every collection name, file title, creation
|
||||
* date, GPS coordinate, camera model, caption, face label, and any
|
||||
* other metadata the Ente clients have attached.
|
||||
*
|
||||
* Layout:
|
||||
*
|
||||
* <dir>/
|
||||
* account.json { email, userID }
|
||||
* collections/
|
||||
* <id>-<sanitized-name>/
|
||||
* _collection.json { id, name, type, magicMetadata?, ... }
|
||||
* <fileID>.json { id, metadata, magicMetadata?, pubMagicMetadata? }
|
||||
*
|
||||
* The test builds a mock server with two collections, each with files
|
||||
* that have different combinations of metadata layers, and verifies
|
||||
* the output tree is correct and complete.
|
||||
*/
|
||||
|
||||
import {
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
} from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import sodium from "libsodium-wrappers-sumo";
|
||||
import { SRP, SrpServer } from "fast-srp-hap";
|
||||
import { beforeAll, afterAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
init,
|
||||
toBase64,
|
||||
deriveKEK,
|
||||
deriveLoginSubkey,
|
||||
} from "../../src/crypto/index.js";
|
||||
import { Client } from "../../src/client.js";
|
||||
import { runMetadataBackup } from "../../src/metadata-backup.js";
|
||||
import type { KeyAttributes } from "../../src/auth/types.js";
|
||||
|
||||
const TEST_EMAIL = "metabackup@example.com";
|
||||
const TEST_PASSWORD = "metapass";
|
||||
const TEST_OPS = 2;
|
||||
const TEST_MEM = 64 * 1024 * 1024;
|
||||
|
||||
interface MetaMockState {
|
||||
verifier: Buffer;
|
||||
srpAttributes: Record<string, unknown>;
|
||||
keyAttributes: KeyAttributes;
|
||||
encryptedToken: string;
|
||||
collections: Record<string, unknown>[];
|
||||
filesByCollection: Record<number, Record<string, unknown>[]>;
|
||||
}
|
||||
|
||||
let mock: MetaMockState;
|
||||
let testDir: string;
|
||||
|
||||
const encryptSecretbox = (plaintext: Uint8Array, key: Uint8Array) => {
|
||||
const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
||||
const ciphertext = sodium.crypto_secretbox_easy(plaintext, nonce, key);
|
||||
return { ciphertext, nonce };
|
||||
};
|
||||
|
||||
const encryptStreamBlob = (plaintext: Uint8Array, key: Uint8Array) => {
|
||||
const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
||||
const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push(
|
||||
push.state,
|
||||
plaintext,
|
||||
null,
|
||||
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
||||
);
|
||||
return { ciphertext, header: push.header };
|
||||
};
|
||||
|
||||
const buildMetaMock = async (): Promise<MetaMockState> => {
|
||||
const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
|
||||
const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM);
|
||||
const loginSubKeyBytes = deriveLoginSubkey(kek);
|
||||
const srpUserID = "meta-srp";
|
||||
const srpSalt = sodium.randombytes_buf(16);
|
||||
const verifier = SRP.computeVerifier(
|
||||
SRP.params["4096"],
|
||||
Buffer.from(srpSalt),
|
||||
Buffer.from(srpUserID),
|
||||
Buffer.from(loginSubKeyBytes),
|
||||
);
|
||||
|
||||
const masterKey = sodium.randombytes_buf(32);
|
||||
const { ciphertext: encMK, nonce: mkNonce } = encryptSecretbox(
|
||||
masterKey,
|
||||
kek,
|
||||
);
|
||||
const kp = sodium.crypto_box_keypair();
|
||||
const { ciphertext: encSK, nonce: skNonce } = encryptSecretbox(
|
||||
kp.privateKey,
|
||||
masterKey,
|
||||
);
|
||||
const tokenBytes = sodium.randombytes_buf(32);
|
||||
const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey);
|
||||
|
||||
const keyAttributes: KeyAttributes = {
|
||||
kekSalt: toBase64(kekSalt),
|
||||
encryptedKey: toBase64(encMK),
|
||||
keyDecryptionNonce: toBase64(mkNonce),
|
||||
publicKey: toBase64(kp.publicKey),
|
||||
encryptedSecretKey: toBase64(encSK),
|
||||
secretKeyDecryptionNonce: toBase64(skNonce),
|
||||
memLimit: TEST_MEM,
|
||||
opsLimit: TEST_OPS,
|
||||
};
|
||||
|
||||
// Collection 1: "Vacation" with collection-level pubMagicMetadata
|
||||
const ck1 = sodium.crypto_secretbox_keygen();
|
||||
const { ciphertext: encCK1, nonce: ck1N } = encryptSecretbox(
|
||||
ck1,
|
||||
masterKey,
|
||||
);
|
||||
const { ciphertext: encCN1, nonce: cn1N } = encryptSecretbox(
|
||||
new TextEncoder().encode("Vacation"),
|
||||
ck1,
|
||||
);
|
||||
const collPubMagic = JSON.stringify({ coverID: 999, sortBy: "date" });
|
||||
const { ciphertext: encCollPM, header: collPMHeader } = encryptStreamBlob(
|
||||
new TextEncoder().encode(collPubMagic),
|
||||
ck1,
|
||||
);
|
||||
|
||||
const rawColl1 = {
|
||||
id: 10,
|
||||
owner: { id: 42 },
|
||||
encryptedKey: toBase64(encCK1),
|
||||
keyDecryptionNonce: toBase64(ck1N),
|
||||
encryptedName: toBase64(encCN1),
|
||||
nameDecryptionNonce: toBase64(cn1N),
|
||||
type: "album",
|
||||
updationTime: 1700000000000000,
|
||||
pubMagicMetadata: {
|
||||
version: 1,
|
||||
count: 1,
|
||||
data: toBase64(encCollPM),
|
||||
header: toBase64(collPMHeader),
|
||||
},
|
||||
};
|
||||
|
||||
// Collection 2: "Work" with no magic metadata
|
||||
const ck2 = sodium.crypto_secretbox_keygen();
|
||||
const { ciphertext: encCK2, nonce: ck2N } = encryptSecretbox(
|
||||
ck2,
|
||||
masterKey,
|
||||
);
|
||||
const { ciphertext: encCN2, nonce: cn2N } = encryptSecretbox(
|
||||
new TextEncoder().encode("Work"),
|
||||
ck2,
|
||||
);
|
||||
const rawColl2 = {
|
||||
id: 20,
|
||||
owner: { id: 42 },
|
||||
encryptedKey: toBase64(encCK2),
|
||||
keyDecryptionNonce: toBase64(ck2N),
|
||||
encryptedName: toBase64(encCN2),
|
||||
nameDecryptionNonce: toBase64(cn2N),
|
||||
type: "folder",
|
||||
updationTime: 1700000000000000,
|
||||
};
|
||||
|
||||
// File 100 in coll 10: has metadata + pubMagicMetadata
|
||||
const fk1 = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
const { ciphertext: encFK1, nonce: fk1N } = encryptSecretbox(fk1, ck1);
|
||||
const meta1 = JSON.stringify({
|
||||
title: "beach.jpg",
|
||||
fileType: 0,
|
||||
creationTime: 1700000000000000,
|
||||
modificationTime: 1700000000000000,
|
||||
latitude: 35.6762,
|
||||
longitude: 139.6503,
|
||||
});
|
||||
const { ciphertext: encMeta1, header: meta1Header } = encryptStreamBlob(
|
||||
new TextEncoder().encode(meta1),
|
||||
fk1,
|
||||
);
|
||||
const pubMagic1 = JSON.stringify({
|
||||
w: 3000,
|
||||
h: 2000,
|
||||
cameraMake: "SONY",
|
||||
cameraModel: "DSC-RX1RM3",
|
||||
});
|
||||
const { ciphertext: encPM1, header: pm1Header } = encryptStreamBlob(
|
||||
new TextEncoder().encode(pubMagic1),
|
||||
fk1,
|
||||
);
|
||||
|
||||
const rawFile1 = {
|
||||
id: 100,
|
||||
collectionID: 10,
|
||||
ownerID: 42,
|
||||
encryptedKey: toBase64(encFK1),
|
||||
keyDecryptionNonce: toBase64(fk1N),
|
||||
metadata: {
|
||||
encryptedData: toBase64(encMeta1),
|
||||
decryptionHeader: toBase64(meta1Header),
|
||||
},
|
||||
pubMagicMetadata: {
|
||||
version: 1,
|
||||
count: 1,
|
||||
data: toBase64(encPM1),
|
||||
header: toBase64(pm1Header),
|
||||
},
|
||||
file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
|
||||
thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
|
||||
updationTime: 1700000000000000,
|
||||
};
|
||||
|
||||
// File 200 in coll 20: metadata only, no magic metadata
|
||||
const fk2 = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
const { ciphertext: encFK2, nonce: fk2N } = encryptSecretbox(fk2, ck2);
|
||||
const meta2 = JSON.stringify({
|
||||
title: "diagram.png",
|
||||
fileType: 0,
|
||||
creationTime: 1710000000000000,
|
||||
modificationTime: 1710000000000000,
|
||||
});
|
||||
const { ciphertext: encMeta2, header: meta2Header } = encryptStreamBlob(
|
||||
new TextEncoder().encode(meta2),
|
||||
fk2,
|
||||
);
|
||||
|
||||
const rawFile2 = {
|
||||
id: 200,
|
||||
collectionID: 20,
|
||||
ownerID: 42,
|
||||
encryptedKey: toBase64(encFK2),
|
||||
keyDecryptionNonce: toBase64(fk2N),
|
||||
metadata: {
|
||||
encryptedData: toBase64(encMeta2),
|
||||
decryptionHeader: toBase64(meta2Header),
|
||||
},
|
||||
file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
|
||||
thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
|
||||
updationTime: 1710000000000000,
|
||||
};
|
||||
|
||||
return {
|
||||
verifier,
|
||||
srpAttributes: {
|
||||
srpUserID,
|
||||
srpSalt: toBase64(srpSalt),
|
||||
memLimit: TEST_MEM,
|
||||
opsLimit: TEST_OPS,
|
||||
kekSalt: toBase64(kekSalt),
|
||||
isEmailMFAEnabled: false,
|
||||
},
|
||||
keyAttributes,
|
||||
encryptedToken: toBase64(encToken),
|
||||
collections: [rawColl1, rawColl2],
|
||||
filesByCollection: { 10: [rawFile1], 20: [rawFile2] },
|
||||
};
|
||||
};
|
||||
|
||||
const buildMetaFetch = (m: MetaMockState) => {
|
||||
let srpServer: SrpServer;
|
||||
return (async (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
): Promise<Response> => {
|
||||
const url =
|
||||
typeof input === "string"
|
||||
? input
|
||||
: input instanceof URL
|
||||
? input.href
|
||||
: input.url;
|
||||
const path = new URL(url).pathname;
|
||||
const json = (body: unknown) =>
|
||||
new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
|
||||
if (path === "/users/srp/attributes")
|
||||
return json({ attributes: m.srpAttributes });
|
||||
if (path === "/users/srp/create-session") {
|
||||
const body = JSON.parse(init?.body as string);
|
||||
const serverKey = await SRP.genKey();
|
||||
srpServer = new SrpServer(
|
||||
SRP.params["4096"],
|
||||
m.verifier,
|
||||
serverKey,
|
||||
);
|
||||
const B = srpServer.computeB();
|
||||
srpServer.setA(Buffer.from(body.srpA, "base64"));
|
||||
return json({ sessionID: "s1", srpB: B.toString("base64") });
|
||||
}
|
||||
if (path === "/users/srp/verify-session") {
|
||||
const body = JSON.parse(init?.body as string);
|
||||
srpServer.checkM1(Buffer.from(body.srpM1, "base64"));
|
||||
return json({
|
||||
srpM2: srpServer.computeM2().toString("base64"),
|
||||
id: 42,
|
||||
keyAttributes: m.keyAttributes,
|
||||
encryptedToken: m.encryptedToken,
|
||||
});
|
||||
}
|
||||
if (path === "/collections/v2")
|
||||
return json({ collections: m.collections });
|
||||
if (path === "/collections/v2/diff") {
|
||||
const collID = Number(
|
||||
new URL(url).searchParams.get("collectionID"),
|
||||
);
|
||||
return json({
|
||||
diff: m.filesByCollection[collID] ?? [],
|
||||
hasMore: false,
|
||||
});
|
||||
}
|
||||
return new Response("not found", { status: 404 });
|
||||
}) as typeof globalThis.fetch;
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
await init();
|
||||
await sodium.ready;
|
||||
mock = await buildMetaMock();
|
||||
testDir = mkdtempSync(join(tmpdir(), "quak-meta-backup-test-"));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
if (testDir && existsSync(testDir))
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("quak backup-metadata", () => {
|
||||
it("writes account.json with email and userID", async () => {
|
||||
const outDir = join(testDir, "full");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
|
||||
const account = JSON.parse(
|
||||
readFileSync(join(outDir, "account.json"), "utf-8"),
|
||||
);
|
||||
expect(account.email).toBe(TEST_EMAIL);
|
||||
expect(account.userID).toBe(42);
|
||||
});
|
||||
|
||||
it("creates per-collection directories with _collection.json", async () => {
|
||||
const outDir = join(testDir, "collections");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
expect(collDirs.length).toBe(2);
|
||||
|
||||
// Find the Vacation collection dir (prefixed with ID)
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
expect(vacDir).toBeDefined();
|
||||
const collMeta = JSON.parse(
|
||||
readFileSync(
|
||||
join(outDir, "collections", vacDir, "_collection.json"),
|
||||
"utf-8",
|
||||
),
|
||||
);
|
||||
expect(collMeta.id).toBe(10);
|
||||
expect(collMeta.name).toBe("Vacation");
|
||||
expect(collMeta.type).toBe("album");
|
||||
});
|
||||
|
||||
it("decrypts collection-level pubMagicMetadata", async () => {
|
||||
const outDir = join(testDir, "coll-magic");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
const collMeta = JSON.parse(
|
||||
readFileSync(
|
||||
join(outDir, "collections", vacDir, "_collection.json"),
|
||||
"utf-8",
|
||||
),
|
||||
);
|
||||
expect(collMeta.pubMagicMetadata).toBeDefined();
|
||||
expect(collMeta.pubMagicMetadata.coverID).toBe(999);
|
||||
expect(collMeta.pubMagicMetadata.sortBy).toBe("date");
|
||||
});
|
||||
|
||||
it("writes per-file JSON with all three metadata layers", async () => {
|
||||
const outDir = join(testDir, "file-meta");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
const fileMeta = JSON.parse(
|
||||
readFileSync(
|
||||
join(outDir, "collections", vacDir, "100.json"),
|
||||
"utf-8",
|
||||
),
|
||||
);
|
||||
expect(fileMeta.id).toBe(100);
|
||||
expect(fileMeta.metadata.title).toBe("beach.jpg");
|
||||
expect(fileMeta.metadata.latitude).toBeCloseTo(35.6762);
|
||||
expect(fileMeta.pubMagicMetadata.cameraMake).toBe("SONY");
|
||||
expect(fileMeta.pubMagicMetadata.w).toBe(3000);
|
||||
});
|
||||
|
||||
it("handles files with no magic metadata gracefully", async () => {
|
||||
const outDir = join(testDir, "no-magic");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const workDir = collDirs.find((d) => d.includes("Work"))!;
|
||||
const fileMeta = JSON.parse(
|
||||
readFileSync(
|
||||
join(outDir, "collections", workDir, "200.json"),
|
||||
"utf-8",
|
||||
),
|
||||
);
|
||||
expect(fileMeta.id).toBe(200);
|
||||
expect(fileMeta.metadata.title).toBe("diagram.png");
|
||||
expect(fileMeta.pubMagicMetadata).toBeUndefined();
|
||||
expect(fileMeta.magicMetadata).toBeUndefined();
|
||||
});
|
||||
|
||||
it("is incremental: second run does not fail", async () => {
|
||||
const outDir = join(testDir, "incremental");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runMetadataBackup(client, outDir);
|
||||
|
||||
const account = JSON.parse(
|
||||
readFileSync(join(outDir, "account.json"), "utf-8"),
|
||||
);
|
||||
expect(account.email).toBe(TEST_EMAIL);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user