From 54133cfa5f9464e5ec3c4613df5af44a5dbabdac Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 14 Oct 2026 03:38:38 +0200 Subject: [PATCH] Test quak backup end to end against a fake Ente server (closes #186) test/cli/backup-end-to-end.test.ts runs backupCommand through run, as bin/quak.ts does, with the real loadSession reading a session saved after logging in, and every request answered by a fake of the Ente HTTP API serving data encrypted with libsodium. It covers the backup tree of an account with two albums of its own and one shared with it, a second run that downloads nothing, a download that succeeds after two 503s, a 404 recorded in failures.json with exit 1, a refresh answered 401 with exit 3, and --verify putting back an original changed on disk. The fake server and the account it logs in moved from the backup-metadata test into test/ente-server.ts, unchanged, so both tests use them. Model: opus-5-5 --- README.md | 3 +- TODO.md | 11 + test/cli/backup-end-to-end.test.ts | 635 +++++++++++++++++++++++++++++ test/cli/metadata-backup.test.ts | 187 +-------- test/ente-server.ts | 197 +++++++++ 5 files changed, 859 insertions(+), 174 deletions(-) create mode 100644 test/cli/backup-end-to-end.test.ts create mode 100644 test/ente-server.ts diff --git a/README.md b/README.md index 32a0b9d..fe40038 100644 --- a/README.md +++ b/README.md @@ -850,7 +850,8 @@ The library's primary surface is the cache-backed `Library`; the lower-level `Client` sits underneath it and is covered by the Design sections above. The test suite is the canonical, executable documentation — `test/library/` and `test/client/usage.test.ts` walk most operations, `test/cli/backup.test.ts` -walks `lib.backup()`, and `yarn test` verifies them. +walks `lib.backup()`, `test/cli/backup-end-to-end.test.ts` runs `quak backup` +against a fake of the Ente HTTP API, and `yarn test` verifies them. ### Opening a library diff --git a/TODO.md b/TODO.md index bacb24a..87de288 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,17 @@ declares one. # Completed Steps +- 2026-10-14: `test/cli/backup-end-to-end.test.ts` runs `quak backup` as cron + does (issue 186): `backupCommand` and `run`, the real `loadSession` reading a + session saved after logging in, and every request answered by a fake of the + Ente HTTP API serving data encrypted with libsodium. It covers the backup tree + of an account with two albums of its own and one shared with it, a second run + that downloads nothing, a download that succeeds after two 503s, a 404 that + lands in `failures.json` with exit 1, a refresh answered 401 with exit 3, and + `--verify` putting back an original changed on disk. The fake server and the + account it logs in moved from the backup-metadata test into + `test/ente-server.ts`, unchanged. + - 2026-10-14: `files --collection`, `get`, `get-thumb` and `helper fix-missing-thumbnails --file` take an ID only when it is written in decimal digits and is a safe integer (issue 185). Before, `Number()` read diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts new file mode 100644 index 0000000..123c00a --- /dev/null +++ b/test/cli/backup-end-to-end.test.ts @@ -0,0 +1,635 @@ +/** + * `quak backup`, end to end. + * + * Each test runs the command as cron runs it: `backupCommand` reads the + * session file that logging in saved, opens the library on the real `Client` + * and backs the whole account up into a directory, and `run` turns the outcome + * into the exit code, as `bin/quak.ts` does. Only the server is a stand-in: + * `buildMetaFetch` from `test/ente-server.ts` answers the Ente HTTP API from + * data this file encrypts with libsodium, as Ente's clients encrypt it. So + * quak's own code fetches each original, decrypts it, checks it against the + * hash its uploader recorded and unpacks a live photo, and quak's own retries + * repeat a failed request. + * + * The account, user 42, has three albums: + * + * Trips its own: beach.jpg + * Family its own: beach.jpg and the live photo IMG_0002.HEIC + * Friend's Wedding shared with it by user 99: toast.mp4 + * + * beach.jpg is one file in two albums. It holds EXIF (a Canon EOS R5), and + * Ente has ML data for it. All three files were taken at noon on 1 March 2024. + */ + +import { + existsSync, + lstatSync, + mkdtempSync, + readdirSync, + readFileSync, + readlinkSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Writable } from "node:stream"; +import { gzipSync } from "node:zlib"; +import sodium from "libsodium-wrappers-sumo"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { + backupCommand, + saveSession, + type CliContext, +} from "../../src/cli-commands.js"; +import { run } from "../../src/cli-run.js"; +import { loadSession } from "../../src/cli-session.js"; +import { Client } from "../../src/client.js"; +import { init, toBase64 } from "../../src/crypto/index.js"; +import { UNATTENDED_RETRY_OPTIONS } from "../../src/retry.js"; +import { + buildAccount, + buildMetaFetch, + encryptSecretbox, + encryptStreamBlob, +} from "../ente-server.js"; +import { HEIC_WITH_EXIF } from "../exif-heic.js"; +import { JPEG_WITH_EXIF } from "../exif-jpeg.js"; +import { blake2b, livePhotoHash, livePhotoZip, VIDEO } from "../live-photo.js"; + +const EMAIL = "backup@example.com"; +const PASSWORD = "backup-password"; +// The fake server logs every account in as user 42. +const USER_ID = 42; +const FRIEND_ID = 99; + +// Noon on 1 March 2024 in the time zone of the machine running the test, the +// one quak dates save paths in, in microseconds, as Ente records times. +const TAKEN = new Date(2024, 2, 1, 12).getTime() * 1000; +// The date folder every original is saved in. +const DAY = "2024/2024-03/2024-03-01"; + +const TOAST = new TextEncoder().encode("a few seconds of the wedding toast"); + +// Ente's ML data for beach.jpg: a face it found, and the photo's CLIP +// embedding. +const ML_DATA = { + face: { faces: [{ faceID: "100_1", score: 0.98 }] }, + clip: { embedding: [0.5, 0.6, 0.7] }, +}; + +// Each file's ID, its owner, its original as uploaded, and the metadata its +// uploader recorded. `fileType` is Ente's number for the kind: 0 an image, 1 a +// video, 2 a live photo, whose original is a ZIP of its image and its video. +// `hash` is the content hash a download must match. +const BEACH = { + id: 100, + ownerID: USER_ID, + original: JPEG_WITH_EXIF, + metadata: { + title: "beach.jpg", + fileType: 0, + hash: blake2b(JPEG_WITH_EXIF), + }, +}; +const TOAST_VIDEO = { + id: 200, + ownerID: FRIEND_ID, + original: TOAST, + metadata: { title: "toast.mp4", fileType: 1, hash: blake2b(TOAST) }, +}; +const LIVE_PHOTO = { + id: 300, + ownerID: USER_ID, + original: livePhotoZip({ + "image.heic": HEIC_WITH_EXIF, + "video.mov": VIDEO, + }), + metadata: { + title: "IMG_0002.HEIC", + fileType: 2, + hash: livePhotoHash(HEIC_WITH_EXIF, VIDEO), + }, +}; + +// The fake Ente server, as the `fetch` quak sends its requests through. +let server: typeof globalThis.fetch; +let root: string; +let sessionDir: string; + +// The fake server, holding the account encrypted as Ente's clients upload it. +const buildServer = async () => { + const account = await buildAccount(PASSWORD); + + // An album as `/collections/v2` lists it, its name encrypted with its own + // key. An album of the account's own carries that key encrypted with the + // master key; an album shared with it carries the key sealed to its public + // key, and no nonce. + const album = (id: number, name: string, ownerID: number) => { + const key = sodium.crypto_secretbox_keygen(); + const encryptedName = encryptSecretbox( + new TextEncoder().encode(name), + key, + ); + let keyFields: { encryptedKey: string; keyDecryptionNonce?: string }; + if (ownerID === USER_ID) { + const encryptedKey = encryptSecretbox(key, account.masterKey); + keyFields = { + encryptedKey: toBase64(encryptedKey.ciphertext), + keyDecryptionNonce: toBase64(encryptedKey.nonce), + }; + } else { + const sealedKey = sodium.crypto_box_seal(key, account.publicKey); + keyFields = { encryptedKey: toBase64(sealedKey) }; + } + const row = { + id, + owner: { id: ownerID }, + ...keyFields, + encryptedName: toBase64(encryptedName.ciphertext), + nameDecryptionNonce: toBase64(encryptedName.nonce), + type: "album", + updationTime: TAKEN, + }; + return { key, row }; + }; + const trips = album(10, "Trips", USER_ID); + const family = album(20, "Family", USER_ID); + const wedding = album(30, "Friend's Wedding", FRIEND_ID); + + // Each file's original is encrypted with the file's own key, as one + // secretstream, which the server serves whole. + const encrypted = (file: typeof BEACH) => { + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + return { file, key, content: encryptStreamBlob(file.original, key) }; + }; + const beach = encrypted(BEACH); + const toast = encrypted(TOAST_VIDEO); + const livePhoto = encrypted(LIVE_PHOTO); + + // A file as an album's `/collections/v2/diff` lists it: its key encrypted + // with the album's key, its metadata encrypted with its own key, and the + // header that starts the decryption of its original. + const inAlbum = ( + { file, key, content }: ReturnType, + collectionID: number, + albumKey: Uint8Array, + ) => { + const encryptedKey = encryptSecretbox(key, albumKey); + const metadata = encryptStreamBlob( + new TextEncoder().encode( + JSON.stringify({ + ...file.metadata, + creationTime: TAKEN, + modificationTime: TAKEN, + }), + ), + key, + ); + return { + id: file.id, + collectionID, + ownerID: file.ownerID, + encryptedKey: toBase64(encryptedKey.ciphertext), + keyDecryptionNonce: toBase64(encryptedKey.nonce), + metadata: { + encryptedData: toBase64(metadata.ciphertext), + decryptionHeader: toBase64(metadata.header), + }, + file: { decryptionHeader: toBase64(content.header) }, + thumbnail: { + decryptionHeader: toBase64(sodium.randombytes_buf(24)), + }, + updationTime: TAKEN, + }; + }; + + // ML data is gzipped JSON, encrypted with the file's key. + const mlData = encryptStreamBlob( + gzipSync(JSON.stringify(ML_DATA)), + beach.key, + ); + + return buildMetaFetch({ + verifier: account.verifier, + srpAttributes: account.srpAttributes, + keyAttributes: account.keyAttributes, + encryptedToken: account.encryptedToken, + collections: [trips.row, family.row, wedding.row], + filesByCollection: { + 10: [inAlbum(beach, 10, trips.key)], + 20: [ + inAlbum(beach, 20, family.key), + inAlbum(livePhoto, 20, family.key), + ], + 30: [inAlbum(toast, 30, wedding.key)], + }, + encryptedMLData: { + 100: { + encryptedData: toBase64(mlData.ciphertext), + decryptionHeader: toBase64(mlData.header), + }, + }, + fileCiphertexts: { + 100: beach.content.ciphertext, + 200: toast.content.ciphertext, + 300: livePhoto.content.ciphertext, + }, + }); +}; + +beforeAll(async () => { + await init(); + await sodium.ready; + root = mkdtempSync(join(tmpdir(), "quak-backup-end-to-end-")); + server = await buildServer(); + + // Log in once, as `quak login` does: the session file it saves is what + // every backup below reads. + sessionDir = join(root, "session"); + const client = await Client.login({ + email: EMAIL, + password: PASSWORD, + apiOptions: { fetch: server }, + }); + saveSession(sessionDir, client.toJSON()); +}); + +afterAll(() => { + rmSync(root, { recursive: true, force: true }); +}); + +// Run `quak backup ` as `bin/quak.ts` runs it, sending its requests +// through `fetch`, and return its exit code and everything it printed on +// stderr. Its requests are retried as those of `quak backup` are, except that +// a retry does not wait. Its cache is beside `dir`, so each run of one +// directory finds the cache the run before it left, as cron's runs find the +// user's. +const backup = async ( + dir: string, + opts: { verify?: boolean } = {}, + fetch = server, +): Promise<{ code: number; stderr: string }> => { + let printed = ""; + const stderr = new Writable({ + write(chunk: Buffer, _encoding, done) { + printed += chunk.toString(); + done(); + }, + }); + const stdout = new Writable({ + write(_chunk, _encoding, done) { + done(); + }, + }); + // A backup never prompts: any of these ends the run with exit 1. + const prompted = async (): Promise => { + throw new Error("quak backup prompted"); + }; + const ctx: CliContext = { + stdout, + stderr, + sessionDir, + cacheDir: `${dir}-cache`, + loadSession: (path) => + loadSession(path, { + fetch, + retry: { ...UNATTENDED_RETRY_OPTIONS, sleep: async () => {} }, + }), + login: prompted, + prompt: prompted, + promptSecret: prompted, + }; + const code = await new Promise((resolve) => { + void run(backupCommand(ctx, dir, opts), stdout, stderr, resolve); + }); + return { code, stderr: printed }; +}; + +// The ID of the file whose original a request downloads, or undefined for any +// other request. quak downloads an original from +// `https://files.ente.io/?fileID=`. +const downloadOf = (input: RequestInfo | URL): number | undefined => { + const url = new URL(String(input)); + return url.hostname === "files.ente.io" + ? Number(url.searchParams.get("fileID")) + : undefined; +}; + +// The fake server, pushing onto `downloads` the ID of each file whose original +// is downloaded. +const recordingDownloads = + (downloads: number[]): typeof globalThis.fetch => + async (input, init) => { + const fileID = downloadOf(input); + if (fileID !== undefined) downloads.push(fileID); + return server(input, init); + }; + +// Every entry under `dir`, relative to it, sorted, a symlink as +// `name -> target`. +const listTree = (dir: string): string[] => + readdirSync(dir, { recursive: true }) + .sort() + .map((name) => { + const path = join(dir, name); + return lstatSync(path).isSymbolicLink() + ? `${name} -> ${readlinkSync(path)}` + : name; + }); + +const readJSON = (path: string) => JSON.parse(readFileSync(path, "utf-8")); + +describe("the first backup of the account", () => { + let dir: string; + let code: number; + const downloads: number[] = []; + + beforeAll(async () => { + dir = mkdtempSync(join(root, "backup-")); + ({ code } = await backup(dir, {}, recordingDownloads(downloads))); + }); + + it("exits 0, having downloaded each file once", () => { + expect(code).toBe(0); + // beach.jpg is in two albums but is downloaded, and stored, once. + expect(downloads.sort()).toEqual([100, 200, 300]); + }); + + // The tree "Backup layout" in the README describes. Each original is at + // its save path, `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD..`, with + // the extension of its title; the live photo is its image and its video, + // each with the extension it had inside the ZIP, and a `.livephoto.json` + // naming the two. Beside each original is its JSON. Each album is a + // folder of links named after the files' titles, with a JSON file beside + // it. No file failed, so there is no `failures.json`, and the run removed + // its lock, `backup.lock`, as it ended. + it("lays the backup out as the README describes", () => { + expect(listTree(dir)).toEqual([ + "2024", + "2024/2024-03", + "2024/2024-03/2024-03-01", + "2024/2024-03/2024-03-01/2024-03-01.100.jpg", + "2024/2024-03/2024-03-01/2024-03-01.100.json", + "2024/2024-03/2024-03-01/2024-03-01.200.json", + "2024/2024-03/2024-03-01/2024-03-01.200.mp4", + "2024/2024-03/2024-03-01/2024-03-01.300.heic", + "2024/2024-03/2024-03-01/2024-03-01.300.json", + "2024/2024-03/2024-03-01/2024-03-01.300.livephoto.json", + "2024/2024-03/2024-03-01/2024-03-01.300.mov", + "account.json", + "collections", + "collections/Family", + "collections/Family.json", + "collections/Family/IMG_0002.heic -> ../../2024/2024-03/2024-03-01/2024-03-01.300.heic", + "collections/Family/IMG_0002.mov -> ../../2024/2024-03/2024-03-01/2024-03-01.300.mov", + "collections/Family/beach.jpg -> ../../2024/2024-03/2024-03-01/2024-03-01.100.jpg", + "collections/Friend's Wedding", + "collections/Friend's Wedding.json", + "collections/Friend's Wedding/toast.mp4 -> ../../2024/2024-03/2024-03-01/2024-03-01.200.mp4", + "collections/Trips", + "collections/Trips.json", + "collections/Trips/beach.jpg -> ../../2024/2024-03/2024-03-01/2024-03-01.100.jpg", + ]); + }); + + it("stores each original decrypted, a live photo as its image and its video", () => { + const day = join(dir, DAY); + expect(readFileSync(join(day, "2024-03-01.100.jpg"))).toEqual( + Buffer.from(JPEG_WITH_EXIF), + ); + expect(readFileSync(join(day, "2024-03-01.200.mp4"))).toEqual( + Buffer.from(TOAST), + ); + expect(readFileSync(join(day, "2024-03-01.300.heic"))).toEqual( + Buffer.from(HEIC_WITH_EXIF), + ); + expect(readFileSync(join(day, "2024-03-01.300.mov"))).toEqual( + Buffer.from(VIDEO), + ); + expect(readJSON(join(day, "2024-03-01.300.livephoto.json"))).toEqual({ + image: "2024-03-01.300.heic", + video: "2024-03-01.300.mov", + }); + }); + + // A file's JSON holds what Ente records for it, decrypted, and what quak + // read from its original: its ML data when Ente has any, and, for an + // image, its EXIF. A video gets no image metadata. + it("writes each file's JSON with its ML data and image metadata", () => { + const day = join(dir, DAY); + + const beach = readJSON(join(day, "2024-03-01.100.json")); + expect(beach).toMatchObject({ + id: 100, + ownerID: USER_ID, + metadata: { + title: "beach.jpg", + fileType: "image", + creationTime: TAKEN, + hash: BEACH.metadata.hash, + }, + updationTime: TAKEN, + }); + expect(beach.mlData).toEqual(ML_DATA); + expect(beach.imageMetadata.exif.Make.description).toBe("Canon"); + expect(beach.imageMetadata.exif.Model.description).toBe("EOS R5"); + + const toast = readJSON(join(day, "2024-03-01.200.json")); + expect(toast).toMatchObject({ + id: 200, + ownerID: FRIEND_ID, + metadata: { title: "toast.mp4", fileType: "video" }, + }); + expect(toast).not.toHaveProperty("mlData"); + expect(toast).not.toHaveProperty("imageMetadata"); + expect(toast).not.toHaveProperty("imageMetadataError"); + + // A live photo's image metadata is read from its image. + const livePhoto = readJSON(join(day, "2024-03-01.300.json")); + expect(livePhoto).toMatchObject({ + id: 300, + ownerID: USER_ID, + metadata: { + title: "IMG_0002.HEIC", + fileType: "livePhoto", + hash: LIVE_PHOTO.metadata.hash, + }, + }); + expect(livePhoto.imageMetadata.exif.Make.description).toBe("Canon"); + }); + + it("writes each album's JSON and the account's", () => { + const albumJSON = (name: string) => + readJSON(join(dir, "collections", `${name}.json`)); + const fileIDs = (album: { files: { id: number }[] }) => + album.files.map((f) => f.id); + + const trips = albumJSON("Trips"); + expect(trips).toMatchObject({ + id: 10, + name: "Trips", + type: "album", + ownerID: USER_ID, + isShared: false, + updationTime: TAKEN, + }); + expect(fileIDs(trips)).toEqual([100]); + + const family = albumJSON("Family"); + expect(family).toMatchObject({ id: 20, isShared: false }); + expect(fileIDs(family)).toEqual([100, 300]); + + const wedding = albumJSON("Friend's Wedding"); + expect(wedding).toMatchObject({ + id: 30, + ownerID: FRIEND_ID, + isShared: true, + }); + expect(fileIDs(wedding)).toEqual([200]); + + expect(readJSON(join(dir, "account.json"))).toEqual({ + email: EMAIL, + userID: USER_ID, + }); + }); +}); + +describe("quak backup", () => { + // An original already at its save path is complete, so a later run, such + // as the next night's from cron, only rewrites the JSON files and the + // album links. + it("downloads nothing on a second run, and exits 0", async () => { + const dir = mkdtempSync(join(root, "backup-")); + expect((await backup(dir)).code).toBe(0); + const firstTree = listTree(dir); + + const downloads: number[] = []; + const { code } = await backup(dir, {}, recordingDownloads(downloads)); + expect(code).toBe(0); + expect(downloads).toEqual([]); + expect(listTree(dir)).toEqual(firstTree); + }); + + // The files CDN is briefly unavailable for toast.mp4: its first two + // downloads answer 503. A 5xx is worth repeating, so quak tries again, + // here without waiting, and the third download succeeds. + it("stores a file whose download fails twice with 503 and then succeeds, and exits 0", async () => { + let attempts = 0; + const fetch: typeof globalThis.fetch = async (input, init) => { + if (downloadOf(input) === 200 && ++attempts <= 2) { + return new Response("unavailable", { status: 503 }); + } + return server(input, init); + }; + const dir = mkdtempSync(join(root, "backup-")); + + const { code } = await backup(dir, {}, fetch); + + expect(code).toBe(0); + expect(attempts).toBe(3); + expect(readFileSync(join(dir, DAY, "2024-03-01.200.mp4"))).toEqual( + Buffer.from(TOAST), + ); + expect(existsSync(join(dir, "failures.json"))).toBe(false); + }); + + // The server answers 404 for toast.mp4 on every try. A 404 is an answer, + // not a hiccup, so quak does not repeat it: the file goes into + // `failures.json` with the kind of failure, how many times it has been + // tried and why, the backup goes on with the other files, and the exit + // code is 1, so the next run tries again. + it("records a file whose download always answers 404 in failures.json, stores the rest, and exits 1", async () => { + let attempts = 0; + const fetch: typeof globalThis.fetch = async (input, init) => { + if (downloadOf(input) === 200) { + attempts++; + return new Response("not found", { status: 404 }); + } + return server(input, init); + }; + const dir = mkdtempSync(join(root, "backup-")); + + const { code, stderr } = await backup(dir, {}, fetch); + + expect(code).toBe(1); + expect(attempts).toBe(1); + const failures = readJSON(join(dir, "failures.json")); + expect(Object.keys(failures.files)).toEqual(["200"]); + expect(failures.files["200"]).toMatchObject({ + fileID: 200, + title: "toast.mp4", + classification: "permanent", + attempts: 1, + }); + expect(failures.files["200"].error).toContain("404"); + expect(stderr).toContain("[Friend's Wedding] toast.mp4 (id 200): "); + + // The rest of the account is backed up; the video is not, and its + // album folder has no link to it. + expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual( + Buffer.from(JPEG_WITH_EXIF), + ); + expect(readFileSync(join(dir, DAY, "2024-03-01.300.heic"))).toEqual( + Buffer.from(HEIC_WITH_EXIF), + ); + expect(existsSync(join(dir, DAY, "2024-03-01.200.mp4"))).toBe(false); + expect( + readdirSync(join(dir, "collections", "Friend's Wedding")), + ).toEqual([]); + }); + + // The server no longer accepts the saved session's token, as after + // `quak logout` elsewhere, and answers every request with 401. The run + // stops at the refresh it starts with, says to log in again, and exits 3. + // It took its lock first, so every request was sent while the lock was + // there, and it removed the lock as it ended: the directory is left as it + // was, empty. + it("exits 3 when the refresh is answered with 401, writing nothing but the lock", async () => { + const dir = mkdtempSync(join(root, "backup-")); + const lockAtEachRequest: boolean[] = []; + const fetch: typeof globalThis.fetch = async () => { + lockAtEachRequest.push(existsSync(join(dir, "backup.lock"))); + return new Response(null, { status: 401 }); + }; + + const { code, stderr } = await backup(dir, {}, fetch); + + expect(code).toBe(3); + expect(stderr).toBe( + "Starting backup...\n" + + "Refreshing library...\n" + + `quak: the saved session is no longer valid; run "quak login"\n`, + ); + expect(lockAtEachRequest).not.toHaveLength(0); + expect(lockAtEachRequest).not.toContain(false); + expect(readdirSync(dir)).toEqual([]); + }); + + // Something changed beach.jpg on disk after it was backed up. A run with + // --verify hashes every stored original, finds that one no longer matches + // the hash Ente recorded, says so, and downloads it again; it put back the + // right bytes, so the exit code is 0. + it("--verify puts back an original that changed on disk, and exits 0", async () => { + const dir = mkdtempSync(join(root, "backup-")); + expect((await backup(dir)).code).toBe(0); + const beach = join(dir, DAY, "2024-03-01.100.jpg"); + writeFileSync(beach, "not the photo any more"); + + const downloads: number[] = []; + const { code, stderr } = await backup( + dir, + { verify: true }, + recordingDownloads(downloads), + ); + + expect(code).toBe(0); + expect(downloads).toEqual([100]); + expect(readFileSync(beach)).toEqual(Buffer.from(JPEG_WITH_EXIF)); + expect(stderr).toContain( + "MISMATCH original beach.jpg (100): its bytes do not match its content hash", + ); + expect(stderr).toContain(" Verified: 2\n Mismatched: 1\n"); + }); +}); diff --git a/test/cli/metadata-backup.test.ts b/test/cli/metadata-backup.test.ts index 6d597e5..5bff19c 100644 --- a/test/cli/metadata-backup.test.ts +++ b/test/cli/metadata-backup.test.ts @@ -37,15 +37,8 @@ import { import { join } from "node:path"; import { tmpdir } from "node:os"; import sodium from "libsodium-wrappers-sumo"; -import { SRP, SrpServer } from "fast-srp-hap"; import { beforeAll, afterAll, describe, expect, it, vi } from "vitest"; -import { - init, - toBase64, - deriveKEK, - deriveLoginSubkey, - encryptBlob, -} from "../../src/crypto/index.js"; +import { init, toBase64, encryptBlob } from "../../src/crypto/index.js"; import * as jpegJs from "jpeg-js"; import { Client } from "../../src/client.js"; import { Library } from "../../src/library/index.js"; @@ -54,7 +47,13 @@ import { type MetadataBackupOptions, } from "../../src/metadata-backup.js"; import { backupMetadataCommand } from "../../src/cli-commands.js"; -import type { KeyAttributes } from "../../src/auth/types.js"; +import { + buildAccount, + buildMetaFetch, + encryptSecretbox, + encryptStreamBlob, + type MetaMockState, +} from "../ente-server.js"; // One file per ML data request, so the two files of the mock account are // fetched in two requests and one of them can fail on its own. @@ -65,80 +64,13 @@ vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({ const TEST_EMAIL = "metabackup@example.com"; const TEST_PASSWORD = "metapass"; -const TEST_OPS = 2; -const TEST_MEM = 64 * 1024 * 1024; - -interface MetaMockState { - verifier: Buffer; - srpAttributes: Record; - keyAttributes: KeyAttributes; - encryptedToken: string; - collections: Record[]; - filesByCollection: Record[]>; - // For ML data and EXIF tests - encryptedMLData: Record< - number, - { encryptedData: string; decryptionHeader: string } - >; - fileCiphertexts: Record; -} let mock: MetaMockState; let testDir: string; -const encryptSecretbox = (plaintext: Uint8Array, key: Uint8Array) => { - const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const ciphertext = sodium.crypto_secretbox_easy(plaintext, nonce, key); - return { ciphertext, nonce }; -}; - -const encryptStreamBlob = (plaintext: Uint8Array, key: Uint8Array) => { - const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key); - const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push( - push.state, - plaintext, - null, - sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, - ); - return { ciphertext, header: push.header }; -}; - const buildMetaMock = async (): Promise => { - const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES); - const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM); - const loginSubKeyBytes = deriveLoginSubkey(kek); - const srpUserID = "meta-srp"; - const srpSalt = sodium.randombytes_buf(16); - const verifier = SRP.computeVerifier( - SRP.params["4096"], - Buffer.from(srpSalt), - Buffer.from(srpUserID), - Buffer.from(loginSubKeyBytes), - ); - - const masterKey = sodium.randombytes_buf(32); - const { ciphertext: encMK, nonce: mkNonce } = encryptSecretbox( - masterKey, - kek, - ); - const kp = sodium.crypto_box_keypair(); - const { ciphertext: encSK, nonce: skNonce } = encryptSecretbox( - kp.privateKey, - masterKey, - ); - const tokenBytes = sodium.randombytes_buf(32); - const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey); - - const keyAttributes: KeyAttributes = { - kekSalt: toBase64(kekSalt), - encryptedKey: toBase64(encMK), - keyDecryptionNonce: toBase64(mkNonce), - publicKey: toBase64(kp.publicKey), - encryptedSecretKey: toBase64(encSK), - secretKeyDecryptionNonce: toBase64(skNonce), - memLimit: TEST_MEM, - opsLimit: TEST_OPS, - }; + const account = await buildAccount(TEST_PASSWORD); + const masterKey = account.masterKey; // Collection 1: "Vacation" with collection-level pubMagicMetadata const ck1 = sodium.crypto_secretbox_keygen(); @@ -332,17 +264,10 @@ const buildMetaMock = async (): Promise => { rawFile1.file.decryptionHeader = toBase64(filePush1.header); return { - verifier, - srpAttributes: { - srpUserID, - srpSalt: toBase64(srpSalt), - memLimit: TEST_MEM, - opsLimit: TEST_OPS, - kekSalt: toBase64(kekSalt), - isEmailMFAEnabled: false, - }, - keyAttributes, - encryptedToken: toBase64(encToken), + verifier: account.verifier, + srpAttributes: account.srpAttributes, + keyAttributes: account.keyAttributes, + encryptedToken: account.encryptedToken, collections: [rawColl1, rawColl2], filesByCollection: { 10: [rawFile1], 20: [rawFile2] }, encryptedMLData: { @@ -355,90 +280,6 @@ const buildMetaMock = async (): Promise => { }; }; -// `failMLDataFor`: answer 500 to every ML data request that asks for this file. -const buildMetaFetch = (m: MetaMockState, failMLDataFor?: number) => { - let srpServer: SrpServer; - return (async ( - input: RequestInfo | URL, - init?: RequestInit, - ): Promise => { - const url = - typeof input === "string" - ? input - : input instanceof URL - ? input.href - : input.url; - const path = new URL(url).pathname; - const json = (body: unknown) => - new Response(JSON.stringify(body), { - status: 200, - headers: { "content-type": "application/json" }, - }); - - if (path === "/users/srp/attributes") - return json({ attributes: m.srpAttributes }); - if (path === "/users/srp/create-session") { - const body = JSON.parse(init?.body as string); - const serverKey = await SRP.genKey(); - srpServer = new SrpServer( - SRP.params["4096"], - m.verifier, - serverKey, - ); - const B = srpServer.computeB(); - srpServer.setA(Buffer.from(body.srpA, "base64")); - return json({ sessionID: "s1", srpB: B.toString("base64") }); - } - if (path === "/users/srp/verify-session") { - const body = JSON.parse(init?.body as string); - srpServer.checkM1(Buffer.from(body.srpM1, "base64")); - return json({ - srpM2: srpServer.computeM2().toString("base64"), - id: 42, - keyAttributes: m.keyAttributes, - encryptedToken: m.encryptedToken, - }); - } - if (path === "/collections/v2") - return json({ collections: m.collections }); - if (path === "/collections/v2/diff") { - const collID = Number( - new URL(url).searchParams.get("collectionID"), - ); - return json({ - diff: m.filesByCollection[collID] ?? [], - hasMore: false, - }); - } - if (path === "/files/data/fetch") { - const body = JSON.parse(init?.body as string); - if ((body.fileIDs as number[]).includes(failMLDataFor!)) - return new Response("server error", { status: 500 }); - const data = (body.fileIDs as number[]) - .filter((id: number) => m.encryptedMLData[id]) - .map((id: number) => ({ - fileID: id, - ...m.encryptedMLData[id], - updatedAt: 1700000000000000, - })); - return json({ data }); - } - if ( - url.includes("files.ente.io") || - path.startsWith("/files/download/") - ) { - const parsed = new URL(url); - const fileID = Number( - parsed.searchParams.get("fileID") ?? path.split("/").pop(), - ); - const ct = m.fileCiphertexts[fileID]; - if (ct) return new Response(ct, { status: 200 }); - return new Response("not found", { status: 404 }); - } - return new Response("not found", { status: 404 }); - }) as typeof globalThis.fetch; -}; - beforeAll(async () => { await init(); await sodium.ready; diff --git a/test/ente-server.ts b/test/ente-server.ts new file mode 100644 index 0000000..64b95db --- /dev/null +++ b/test/ente-server.ts @@ -0,0 +1,197 @@ +/** + * A fake of the Ente HTTP API, for tests that run quak's real `Client` against + * a server. + * + * `buildMetaFetch` is a `fetch` that answers the requests quak makes, from a + * `MetaMockState` the test builds: the SRP login, `/collections/v2`, each + * collection's `/collections/v2/diff`, the ML data of `/files/data/fetch`, and + * each file's content from the files CDN, a 404 for a file it holds none for. + * The test encrypts everything in that state with libsodium, as Ente's clients + * do, so quak decrypts it with its own code. Nothing hits the network. + * + * `buildAccount` builds what the server holds to log an account in, and the + * keys a test encrypts that account's collections with. Call `init()` from + * `src/crypto` and await `sodium.ready` before using either. + */ + +import sodium from "libsodium-wrappers-sumo"; +import { SRP, SrpServer } from "fast-srp-hap"; +import { toBase64, deriveKEK, deriveLoginSubkey } from "../src/crypto/index.js"; +import type { KeyAttributes } from "../src/auth/types.js"; + +const TEST_OPS = 2; +const TEST_MEM = 64 * 1024 * 1024; + +export interface MetaMockState { + verifier: Buffer; + srpAttributes: Record; + keyAttributes: KeyAttributes; + encryptedToken: string; + collections: Record[]; + filesByCollection: Record[]>; + // For ML data and EXIF tests + encryptedMLData: Record< + number, + { encryptedData: string; decryptionHeader: string } + >; + fileCiphertexts: Record; +} + +export const encryptSecretbox = (plaintext: Uint8Array, key: Uint8Array) => { + const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + const ciphertext = sodium.crypto_secretbox_easy(plaintext, nonce, key); + return { ciphertext, nonce }; +}; + +export const encryptStreamBlob = (plaintext: Uint8Array, key: Uint8Array) => { + const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key); + const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push( + push.state, + plaintext, + null, + sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, + ); + return { ciphertext, header: push.header }; +}; + +// An account whose password is `password`. The first four fields are those of +// `MetaMockState`; with the master key the test encrypts the keys of the +// account's own collections, and to the public key it seals the keys of the +// collections other accounts share with it. +export const buildAccount = async (password: string) => { + const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES); + const kek = await deriveKEK(password, kekSalt, TEST_OPS, TEST_MEM); + const loginSubKeyBytes = deriveLoginSubkey(kek); + const srpUserID = "meta-srp"; + const srpSalt = sodium.randombytes_buf(16); + const verifier = SRP.computeVerifier( + SRP.params["4096"], + Buffer.from(srpSalt), + Buffer.from(srpUserID), + Buffer.from(loginSubKeyBytes), + ); + + const masterKey = sodium.randombytes_buf(32); + const { ciphertext: encMK, nonce: mkNonce } = encryptSecretbox( + masterKey, + kek, + ); + const kp = sodium.crypto_box_keypair(); + const { ciphertext: encSK, nonce: skNonce } = encryptSecretbox( + kp.privateKey, + masterKey, + ); + const tokenBytes = sodium.randombytes_buf(32); + const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey); + + const keyAttributes: KeyAttributes = { + kekSalt: toBase64(kekSalt), + encryptedKey: toBase64(encMK), + keyDecryptionNonce: toBase64(mkNonce), + publicKey: toBase64(kp.publicKey), + encryptedSecretKey: toBase64(encSK), + secretKeyDecryptionNonce: toBase64(skNonce), + memLimit: TEST_MEM, + opsLimit: TEST_OPS, + }; + + return { + verifier, + srpAttributes: { + srpUserID, + srpSalt: toBase64(srpSalt), + memLimit: TEST_MEM, + opsLimit: TEST_OPS, + kekSalt: toBase64(kekSalt), + isEmailMFAEnabled: false, + }, + keyAttributes, + encryptedToken: toBase64(encToken), + masterKey, + publicKey: kp.publicKey, + }; +}; + +// `failMLDataFor`: answer 500 to every ML data request that asks for this file. +export const buildMetaFetch = (m: MetaMockState, failMLDataFor?: number) => { + let srpServer: SrpServer; + return (async ( + input: RequestInfo | URL, + init?: RequestInit, + ): Promise => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.href + : input.url; + const path = new URL(url).pathname; + const json = (body: unknown) => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); + + if (path === "/users/srp/attributes") + return json({ attributes: m.srpAttributes }); + if (path === "/users/srp/create-session") { + const body = JSON.parse(init?.body as string); + const serverKey = await SRP.genKey(); + srpServer = new SrpServer( + SRP.params["4096"], + m.verifier, + serverKey, + ); + const B = srpServer.computeB(); + srpServer.setA(Buffer.from(body.srpA, "base64")); + return json({ sessionID: "s1", srpB: B.toString("base64") }); + } + if (path === "/users/srp/verify-session") { + const body = JSON.parse(init?.body as string); + srpServer.checkM1(Buffer.from(body.srpM1, "base64")); + return json({ + srpM2: srpServer.computeM2().toString("base64"), + id: 42, + keyAttributes: m.keyAttributes, + encryptedToken: m.encryptedToken, + }); + } + if (path === "/collections/v2") + return json({ collections: m.collections }); + if (path === "/collections/v2/diff") { + const collID = Number( + new URL(url).searchParams.get("collectionID"), + ); + return json({ + diff: m.filesByCollection[collID] ?? [], + hasMore: false, + }); + } + if (path === "/files/data/fetch") { + const body = JSON.parse(init?.body as string); + if ((body.fileIDs as number[]).includes(failMLDataFor!)) + return new Response("server error", { status: 500 }); + const data = (body.fileIDs as number[]) + .filter((id: number) => m.encryptedMLData[id]) + .map((id: number) => ({ + fileID: id, + ...m.encryptedMLData[id], + updatedAt: 1700000000000000, + })); + return json({ data }); + } + if ( + url.includes("files.ente.io") || + path.startsWith("/files/download/") + ) { + const parsed = new URL(url); + const fileID = Number( + parsed.searchParams.get("fileID") ?? path.split("/").pop(), + ); + const ct = m.fileCiphertexts[fileID]; + if (ct) return new Response(ct, { status: 200 }); + return new Response("not found", { status: 404 }); + } + return new Response("not found", { status: 404 }); + }) as typeof globalThis.fetch; +};