diff --git a/package.json b/package.json index 1e84485..c967ef3 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,8 @@ "vitest": "2.1.9" }, "dependencies": { + "commander": "14.0.3", + "env-paths": "4.0.0", "fast-srp-hap": "2.0.4", "libsodium-wrappers-sumo": "0.8.4" } diff --git a/src/backup.ts b/src/backup.ts new file mode 100644 index 0000000..a50d749 --- /dev/null +++ b/src/backup.ts @@ -0,0 +1,25 @@ +// Stub: see the README "Development workflow" section for TDD policy. + +import type { Client } from "./client.js"; + +export interface BackupError { + fileID: number; + title: string; + collection: string; + error: string; +} + +export interface BackupResult { + totalFiles: number; + downloaded: number; + skipped: number; + failed: number; + errors: BackupError[]; +} + +export const runBackup = async ( + _client: Client, + _outDir: string, +): Promise => { + throw new Error("backup.runBackup not implemented"); +}; diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts new file mode 100644 index 0000000..536d658 --- /dev/null +++ b/test/cli/backup.test.ts @@ -0,0 +1,438 @@ +/** + * Tests for the `quak backup` command's core logic. + * + * `quak backup ` downloads every file from every collection into + * a local directory tree: + * + * / + * / + * + * + * / + * ... + * metadata.json (all decrypted collection + file metadata) + * + * The backup command has two properties that distinguish it from a naive + * "download everything" loop: + * + * 1. **Skip existing files.** If `//` already + * exists on disk and its size matches the decrypted content length + * recorded in metadata.json from a prior run, the file is not + * re-downloaded. This makes interrupted backups resumable and + * incremental runs fast. + * + * 2. **Never crash on a single file failure.** If a file download or + * decryption fails, the error is logged and the backup continues + * with the next file. At the end, the exit code is non-zero if any + * files failed, and the summary lists them. The Ente first-party + * CLI crashes entirely when a single file can't be retrieved, + * which defeats the purpose of a backup tool. + * + * These tests exercise the backup logic (in src/backup.ts) using the + * same mock server from the Client usage tests. The CLI binary itself + * is a thin wrapper around this module. + */ + +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import sodium from "libsodium-wrappers-sumo"; +import { SRP, SrpServer } from "fast-srp-hap"; +import { beforeAll, afterAll, describe, expect, it } from "vitest"; +import { + init, + toBase64, + deriveKEK, + deriveLoginSubkey, +} from "../../src/crypto/index.js"; +import { Client } from "../../src/client.js"; +import { runBackup } from "../../src/backup.js"; +import type { KeyAttributes } from "../../src/auth/types.js"; + +// --------------------------------------------------------------------------- +// Mock server (condensed from usage.test.ts) +// --------------------------------------------------------------------------- + +const TEST_EMAIL = "backup@example.com"; +const TEST_PASSWORD = "backuppass"; +const TEST_OPS = 2; +const TEST_MEM = 64 * 1024 * 1024; + +interface MockState { + verifier: Buffer; + srpAttributes: Record<string, unknown>; + keyAttributes: KeyAttributes; + encryptedToken: string; + collections: Record<string, unknown>[]; + files: Record< + number, + { raw: Record<string, unknown>; plaintext: Uint8Array } + >; +} + +let mock: MockState; +let testDir: string; + +const buildMock = async (): Promise<MockState> => { + const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES); + const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM); + const loginSubKeyBytes = deriveLoginSubkey(kek); + + const srpUserID = "backup-srp"; + const srpSalt = sodium.randombytes_buf(16); + const verifier = SRP.computeVerifier( + SRP.params["4096"], + Buffer.from(srpSalt), + Buffer.from(srpUserID), + Buffer.from(loginSubKeyBytes), + ); + + const masterKey = sodium.randombytes_buf(32); + const keyNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + const encryptedKey = sodium.crypto_secretbox_easy(masterKey, keyNonce, kek); + const kp = sodium.crypto_box_keypair(); + const skNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + const encSK = sodium.crypto_secretbox_easy( + kp.privateKey, + skNonce, + masterKey, + ); + const tokenBytes = sodium.randombytes_buf(32); + const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey); + + const keyAttributes: KeyAttributes = { + kekSalt: toBase64(kekSalt), + encryptedKey: toBase64(encryptedKey), + keyDecryptionNonce: toBase64(keyNonce), + publicKey: toBase64(kp.publicKey), + encryptedSecretKey: toBase64(encSK), + secretKeyDecryptionNonce: toBase64(skNonce), + memLimit: TEST_MEM, + opsLimit: TEST_OPS, + }; + + const makeCollection = (id: number, name: string) => { + const ck = sodium.crypto_secretbox_keygen(); + const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + const encCK = sodium.crypto_secretbox_easy(ck, ckN, masterKey); + const nameBytes = new TextEncoder().encode(name); + const cnN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + const encCN = sodium.crypto_secretbox_easy(nameBytes, cnN, ck); + return { + raw: { + id, + owner: { id: 42 }, + encryptedKey: toBase64(encCK), + keyDecryptionNonce: toBase64(ckN), + encryptedName: toBase64(encCN), + nameDecryptionNonce: toBase64(cnN), + type: "album", + updationTime: 1700000000000000, + }, + key: ck, + }; + }; + + const makeFile = ( + id: number, + collKey: Uint8Array, + title: string, + plaintext: Uint8Array, + collID: number, + ) => { + const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey); + + const meta = JSON.stringify({ + title, + fileType: 0, + creationTime: 1700000000000000, + modificationTime: 1700000000000000, + }); + const metaPush = + sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); + const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push( + metaPush.state, + new TextEncoder().encode(meta), + null, + sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, + ); + + const filePush = + sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); + const encFile = sodium.crypto_secretstream_xchacha20poly1305_push( + filePush.state, + plaintext, + null, + sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, + ); + + return { + raw: { + id, + collectionID: collID, + ownerID: 42, + encryptedKey: toBase64(encFK), + keyDecryptionNonce: toBase64(fkN), + metadata: { + encryptedData: toBase64(encMeta), + decryptionHeader: toBase64(metaPush.header), + }, + file: { decryptionHeader: toBase64(filePush.header) }, + thumbnail: { + decryptionHeader: toBase64(sodium.randombytes_buf(24)), + }, + updationTime: 1700000000000000, + }, + plaintext, + ciphertext: encFile, + }; + }; + + const col1 = makeCollection(1, "Vacation"); + const col2 = makeCollection(2, "Work"); + const file1 = makeFile( + 100, + col1.key, + "beach.jpg", + sodium.randombytes_buf(3000), + 1, + ); + const file2 = makeFile( + 101, + col1.key, + "sunset.jpg", + sodium.randombytes_buf(2000), + 1, + ); + const file3 = makeFile( + 200, + col2.key, + "diagram.png", + sodium.randombytes_buf(1500), + 2, + ); + + return { + verifier, + srpAttributes: { + srpUserID, + srpSalt: toBase64(srpSalt), + memLimit: TEST_MEM, + opsLimit: TEST_OPS, + kekSalt: toBase64(kekSalt), + isEmailMFAEnabled: false, + }, + keyAttributes, + encryptedToken: toBase64(encToken), + collections: [col1.raw, col2.raw], + files: { + 1: { + raw: [file1.raw, file2.raw], + ciphertexts: { 100: file1.ciphertext, 101: file2.ciphertext }, + }, + 2: { raw: [file3.raw], ciphertexts: { 200: file3.ciphertext } }, + 100: { plaintext: file1.plaintext }, + 101: { plaintext: file2.plaintext }, + 200: { plaintext: file3.plaintext }, + } as Record<number, unknown>, + }; +}; + +const buildMockFetch = (m: MockState, opts?: { failFileID?: number }) => { + let srpServer: SrpServer; + + return (async ( + input: RequestInfo | URL, + init?: RequestInit, + ): Promise<Response> => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.href + : input.url; + const parsed = new URL(url); + const path = parsed.pathname; + const json = (body: unknown) => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); + + if (path === "/users/srp/attributes") + return json({ attributes: m.srpAttributes }); + + if (path === "/users/srp/create-session") { + const body = JSON.parse(init?.body as string); + const serverKey = await SRP.genKey(); + srpServer = new SrpServer( + SRP.params["4096"], + m.verifier, + serverKey, + ); + const B = srpServer.computeB(); + srpServer.setA(Buffer.from(body.srpA, "base64")); + return json({ sessionID: "s1", srpB: B.toString("base64") }); + } + + if (path === "/users/srp/verify-session") { + const body = JSON.parse(init?.body as string); + srpServer.checkM1(Buffer.from(body.srpM1, "base64")); + return json({ + srpM2: srpServer.computeM2().toString("base64"), + id: 42, + keyAttributes: m.keyAttributes, + encryptedToken: m.encryptedToken, + }); + } + + if (path === "/collections/v2") + return json({ collections: m.collections }); + + if (path === "/collections/v2/diff") { + const collID = Number(parsed.searchParams.get("collectionID")); + const collData = m.files[collID] as { raw: unknown[] } | undefined; + return json({ diff: collData?.raw ?? [], hasMore: false }); + } + + // File download + if (url.includes("fileID=") || path.startsWith("/files/download/")) { + const fileID = Number( + parsed.searchParams.get("fileID") ?? path.split("/").pop(), + ); + if (opts?.failFileID === fileID) { + return new Response("Internal Server Error", { status: 500 }); + } + const collData = Object.values(m.files).find( + (v: unknown) => + v && + typeof v === "object" && + "ciphertexts" in (v as Record<string, unknown>) && + fileID in + (v as Record<string, Record<number, unknown>>) + .ciphertexts, + ) as { ciphertexts: Record<number, Uint8Array> } | undefined; + if (collData) { + return new Response(collData.ciphertexts[fileID], { + status: 200, + }); + } + return new Response("not found", { status: 404 }); + } + + return new Response("not found", { status: 404 }); + }) as typeof globalThis.fetch; +}; + +// --------------------------------------------------------------------------- +// Setup / teardown +// --------------------------------------------------------------------------- + +beforeAll(async () => { + await init(); + await sodium.ready; + mock = await buildMock(); + testDir = mkdtempSync(join(tmpdir(), "quak-backup-test-")); +}); + +afterAll(() => { + if (testDir && existsSync(testDir)) + rmSync(testDir, { recursive: true, force: true }); +}); + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("quak backup", () => { + it("downloads all files organized by collection name", async () => { + const outDir = join(testDir, "full-backup"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMockFetch(mock) }, + }); + + const result = await runBackup(client, outDir); + + expect(result.totalFiles).toBe(3); + expect(result.downloaded).toBe(3); + expect(result.skipped).toBe(0); + expect(result.failed).toBe(0); + expect(result.errors).toEqual([]); + + // Files are under <outDir>/<collection-name>/<title> + const beach = readFileSync(join(outDir, "Vacation", "beach.jpg")); + const sunset = readFileSync(join(outDir, "Vacation", "sunset.jpg")); + const diagram = readFileSync(join(outDir, "Work", "diagram.png")); + expect(beach.length).toBe(3000); + expect(sunset.length).toBe(2000); + expect(diagram.length).toBe(1500); + }); + + it("skips files that already exist on disk with matching size", async () => { + const outDir = join(testDir, "incremental"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMockFetch(mock) }, + }); + + // First run: download everything + const first = await runBackup(client, outDir); + expect(first.downloaded).toBe(3); + + // Second run: everything should be skipped + const second = await runBackup(client, outDir); + expect(second.downloaded).toBe(0); + expect(second.skipped).toBe(3); + expect(second.failed).toBe(0); + }); + + it("continues after a single file download failure", async () => { + // File 101 (sunset.jpg) will return HTTP 500. The other two + // files must still download. The result must report the failure + // without throwing. + const outDir = join(testDir, "partial-failure"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMockFetch(mock, { failFileID: 101 }) }, + }); + + const result = await runBackup(client, outDir); + + expect(result.totalFiles).toBe(3); + expect(result.downloaded).toBe(2); + expect(result.failed).toBe(1); + expect(result.errors.length).toBe(1); + expect(result.errors[0]!.fileID).toBe(101); + expect(result.errors[0]!.title).toBe("sunset.jpg"); + + // The two successful files are on disk + expect(existsSync(join(outDir, "Vacation", "beach.jpg"))).toBe(true); + expect(existsSync(join(outDir, "Work", "diagram.png"))).toBe(true); + // The failed file is NOT on disk (no partial write) + expect(existsSync(join(outDir, "Vacation", "sunset.jpg"))).toBe(false); + }); + + it("writes metadata.json with collection and file info", async () => { + const outDir = join(testDir, "metadata-check"); + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMockFetch(mock) }, + }); + + await runBackup(client, outDir); + + const metaPath = join(outDir, "metadata.json"); + expect(existsSync(metaPath)).toBe(true); + const meta = JSON.parse(readFileSync(metaPath, "utf-8")); + expect(meta.collections.length).toBe(2); + expect(meta.collections[0].files.length).toBeGreaterThan(0); + expect(meta.collections[0].files[0].metadata.title).toBeDefined(); + }); +}); diff --git a/yarn.lock b/yarn.lock index d958d9b..d3f68e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -680,6 +680,11 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +commander@14.0.3: + version "14.0.3" + resolved "https://registry.yarnpkg.com/commander/-/commander-14.0.3.tgz#425d79b48f9af82fcd9e4fc1ea8af6c5ec07bbc2" + integrity sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw== + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -711,6 +716,13 @@ deep-is@^0.1.3: resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== +env-paths@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-4.0.0.tgz#d0bb1f84a81d2542581bf7b7e8085d0683b39097" + integrity sha512-pxP8eL2SwwaTRi/KHYwLYXinDs7gL3jxFcBYmEdYfZmZXbaVDvdppd0XBU8qVz03rDfKZMXg1omHCbsJjZrMsw== + dependencies: + is-safe-filename "^0.1.0" + es-module-lexer@^1.5.4: version "1.7.0" resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-1.7.0.tgz#9159601561880a85f2734560a9099b2c31e5372a" @@ -879,7 +891,7 @@ fast-levenshtein@^2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== -fast-srp-hap@^2.0.4: +fast-srp-hap@2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/fast-srp-hap/-/fast-srp-hap-2.0.4.tgz#9db296e21a5143951310f99e5a74290106467811" integrity sha512-lHRYYaaIbMrhZtsdGTwPN82UbqD9Bv8QfOlKs+Dz6YRnByZifOh93EYmf2iEWFtkOEIqR2IK8cFD0UN5wLIWBQ== @@ -1000,6 +1012,11 @@ is-number@^7.0.0: resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== +is-safe-filename@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/is-safe-filename/-/is-safe-filename-0.1.1.tgz#fb22eead097c614c47aa674de5d79a1648a53e66" + integrity sha512-4SrR7AdnY11LHfDKTZY1u6Ga3RuxZdl3YKWWShO5iyuG5h8QS4GD2tOb04peBJ5I7pXbR+CGBNEhTcwK+FzN3g== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"