diff --git a/test/cli/backup-undecryptable.test.ts b/test/cli/backup-undecryptable.test.ts new file mode 100644 index 0000000..44eee5f --- /dev/null +++ b/test/cli/backup-undecryptable.test.ts @@ -0,0 +1,424 @@ +/** + * `quak backup` over an account holding an album or a file quak cannot decrypt + * (issue 181). + * + * The server sends every album and file encrypted, and a refresh decrypts each + * one it fetches. A record whose key no key of the account opens, or whose + * metadata is not JSON, must not fail the refresh: a refresh that fails commits + * nothing, so `quak backup` would store nothing, on every run, until the + * server's copy changed. Instead the refresh leaves that record out, and the + * library keeps a list of what it left out in its cache. It has to: the next + * refresh asks the server only for what changed since the last one, so it will + * not see the record again. + * + * The backup stores every other file. It records each file it could not + * decrypt in `failures.json`, prints one line for each album it could not + * decrypt, and exits 1 while any remain. Once the server sends a copy that + * decrypts, or deletes the record, the next run drops it and can exit 0. The + * read commands answer from what did decrypt. + * + * These tests run the commands with a real saved session over a stand-in Ente + * server, which serves albums, files and originals encrypted with libsodium as + * the real server does. A record is broken the way a corrupt one arrives: its + * key replaced by bytes no key opens, or its metadata text that is not JSON. + */ + +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import sodium from "libsodium-wrappers-sumo"; +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +import { + backupCommand, + collectionsCommand, + filesCommand, + saveSession, + type CliContext, +} from "../../src/cli-commands.js"; +import { loadSession } from "../../src/cli-session.js"; +import type { ClientSnapshot } from "../../src/client.js"; +import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js"; + +const USER_ID = 42; + +// Every file is taken at noon local time on 2026-03-01, in microseconds as Ente +// stores times, so its original is saved in the folder `DAY` whatever the time +// zone of the machine running the tests. +const TAKEN = new Date(2026, 2, 1, 12).getTime() * 1000; +const DAY = join("2026", "2026-03", "2026-03-01"); + +type Row = { id: number; updationTime: number } & Record; + +// `plaintext` sealed under `key` with a fresh nonce, both base64, as Ente +// sends a key or an album name. +const secretbox = ( + plaintext: Uint8Array, + key: Uint8Array, +): { ciphertext: string; nonce: string } => { + const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + return { + ciphertext: toBase64( + sodium.crypto_secretbox_easy(plaintext, nonce, key), + ), + nonce: toBase64(nonce), + }; +}; + +// What a corrupt row carries in place of its sealed key: random bytes of the +// same length, which no key opens. +const corruptKey = (): string => + toBase64( + sodium.randombytes_buf( + sodium.crypto_secretbox_KEYBYTES + sodium.crypto_secretbox_MACBYTES, + ), + ); + +const json = (body: unknown): Response => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); + +// A stand-in Ente server for one account. It holds each album, and each file in +// an album, as the row `/collections/v2` and `/collections/v2/diff` serve, and +// answers a request with the rows whose `updationTime` is past the request's +// `sinceTime`, as the real server does. A change to a file also moves its +// album's `updationTime`, which is what tells a client to list the album's +// files again. Originals are served by file ID, as the files CDN serves them. +class FakeEnte { + private readonly masterKey = sodium.crypto_secretbox_keygen(); + private readonly keyPair = sodium.crypto_box_keypair(); + private readonly albums = new Map(); + private readonly albumKeys = new Map(); + // Each album's file rows, by album ID and then file ID. + private readonly files = new Map>(); + // Each file's key and its original as served, by file ID. A file in two + // albums is one original under one key, sealed under each album's key. + private readonly originals = new Map< + number, + { key: Uint8Array; header: string; body: Uint8Array } + >(); + + // The session `quak login` would have saved for the account. + snapshot(): ClientSnapshot { + return { + email: "user@example.com", + userID: USER_ID, + token: "token", + masterKey: toBase64(this.masterKey), + secretKey: toBase64(this.keyPair.privateKey), + publicKey: toBase64(this.keyPair.publicKey), + }; + } + + // Album `id`, named `name`, as of time `t`. With `corrupt`, its key cannot + // be decrypted. An album keeps its key from one call to the next, so its + // files' rows still decrypt once it is put back without `corrupt`. + putAlbum(id: number, name: string, t: number, corrupt = false): void { + const key = this.albumKeys.get(id) ?? sodium.crypto_secretbox_keygen(); + this.albumKeys.set(id, key); + const sealedKey = secretbox(key, this.masterKey); + const sealedName = secretbox(new TextEncoder().encode(name), key); + this.albums.set(id, { + id, + owner: { id: USER_ID }, + encryptedKey: corrupt ? corruptKey() : sealedKey.ciphertext, + keyDecryptionNonce: sealedKey.nonce, + encryptedName: sealedName.ciphertext, + nameDecryptionNonce: sealedName.nonce, + type: "album", + updationTime: t, + }); + if (!this.files.has(id)) this.files.set(id, new Map()); + } + + // File `id`, titled `photo-.jpg`, in album `albumID` as of time `t`. + // With `corruptKey`, its key cannot be decrypted. With `metadata`, its + // metadata decrypts to that text instead of the JSON a client uploads. + putFile( + albumID: number, + id: number, + t: number, + opts: { corruptKey?: boolean; metadata?: string } = {}, + ): void { + const original = this.originalOf(id); + const sealedKey = secretbox(original.key, this.albumKeys.get(albumID)!); + const metadata = + opts.metadata ?? + JSON.stringify({ + title: `photo-${id}.jpg`, + fileType: 0, + creationTime: TAKEN, + modificationTime: TAKEN, + }); + const sealedMetadata = encryptBlob( + new TextEncoder().encode(metadata), + original.key, + ); + this.files.get(albumID)!.set(id, { + id, + collectionID: albumID, + ownerID: USER_ID, + encryptedKey: opts.corruptKey ? corruptKey() : sealedKey.ciphertext, + keyDecryptionNonce: sealedKey.nonce, + metadata: { + encryptedData: toBase64(sealedMetadata.ciphertext), + decryptionHeader: toBase64(sealedMetadata.header), + }, + file: { decryptionHeader: original.header }, + thumbnail: { decryptionHeader: original.header }, + updationTime: t, + }); + this.moveAlbum(albumID, t); + } + + deleteFile(albumID: number, id: number, t: number): void { + this.files + .get(albumID)! + .set(id, { id, updationTime: t, isDeleted: true }); + this.moveAlbum(albumID, t); + } + + deleteAlbum(id: number, t: number): void { + this.albums.set(id, { id, updationTime: t, isDeleted: true }); + } + + readonly fetch = (async (input: string | URL | Request) => { + const url = new URL(input instanceof Request ? input.url : input); + const since = Number(url.searchParams.get("sinceTime")); + const changed = (rows: Iterable): Row[] => + [...rows].filter((row) => row.updationTime > since); + switch (url.pathname) { + case "/collections/v2": + return json({ collections: changed(this.albums.values()) }); + case "/collections/v2/diff": { + const albumID = Number(url.searchParams.get("collectionID")); + const rows = this.files.get(albumID)?.values() ?? []; + return json({ diff: changed(rows), hasMore: false }); + } + case "/files/data/fetch": + // No file has ML data. + return json({ data: [] }); + default: { + const fileID = Number(url.searchParams.get("fileID")); + const original = this.originals.get(fileID); + return original + ? new Response(original.body) + : new Response("not found", { status: 404 }); + } + } + }) as typeof globalThis.fetch; + + private originalOf(id: number): { + key: Uint8Array; + header: string; + body: Uint8Array; + } { + const known = this.originals.get(id); + if (known) return known; + const key = sodium.crypto_secretbox_keygen(); + const { header, ciphertext } = encryptBlob( + new TextEncoder().encode(`original of ${id}`), + key, + ); + const original = { key, header: toBase64(header), body: ciphertext }; + this.originals.set(id, original); + return original; + } + + private moveAlbum(albumID: number, t: number): void { + this.albums.set(albumID, { + ...this.albums.get(albumID)!, + updationTime: t, + }); + } +} + +// Album 1, "Trip", holds file 10, which decrypts, file 11, whose key does not, +// and file 12, whose metadata is not JSON. Album 2, "Family", holds file 20, +// but the album's own key does not decrypt. All of it as of time 1. +const brokenAccount = (): FakeEnte => { + const ente = new FakeEnte(); + ente.putAlbum(1, "Trip", 1); + ente.putFile(1, 10, 1); + ente.putFile(1, 11, 1, { corruptKey: true }); + ente.putFile(1, 12, 1, { metadata: "not json" }); + ente.putAlbum(2, "Family", 1, true); + ente.putFile(2, 20, 1); + return ente; +}; + +// Collects everything written to it. +class Output { + text = ""; + write(text: string): void { + this.text += text; + } +} + +let root: string; +let stdout: Output; +let stderr: Output; + +beforeAll(async () => { + await init(); + await sodium.ready; +}); + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-undecryptable-")); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +// A context for one command, with fresh output, whose saved session talks to +// `ente`. Every run shares one cache directory, as runs of `quak` do. +const context = (ente: FakeEnte): CliContext => { + stdout = new Output(); + stderr = new Output(); + saveSession(join(root, "session"), ente.snapshot()); + return { + stdout, + stderr, + sessionDir: join(root, "session"), + cacheDir: join(root, "cache"), + loadSession: (path) => + loadSession(path, { fetch: ente.fetch, retry: { attempts: 1 } }), + login: async () => { + throw new Error("login not expected"); + }, + prompt: async () => { + throw new Error("prompt not expected"); + }, + promptSecret: async () => { + throw new Error("prompt not expected"); + }, + }; +}; + +const backupDir = (): string => join(root, "backup"); + +// Runs `quak backup` into the backup directory and returns its exit code. +const backup = (ente: FakeEnte, opts: { json?: boolean } = {}) => + backupCommand(context(ente), backupDir(), opts); + +// The backup's `failures.json` entries by file ID, or {} when it has none. +const failures = (): Record => { + const path = join(backupDir(), "failures.json"); + return existsSync(path) + ? JSON.parse(readFileSync(path, "utf-8")).files + : {}; +}; + +// Whether file `id`'s original is at its save path in the backup. +const stored = (id: number): boolean => + existsSync(join(backupDir(), DAY, `2026-03-01.${id}.jpg`)); + +describe("quak backup with records it cannot decrypt", () => { + it("backs up every other file, records each file it cannot decrypt in failures.json, names each such album, and exits 1", async () => { + expect(await backup(brokenAccount())).toBe(1); + + expect(stored(10)).toBe(true); + // Without its album's key, file 20 cannot even be listed. + expect(stored(20)).toBe(false); + expect(failures()).toEqual({ + "11": expect.objectContaining({ + fileID: 11, + error: expect.stringMatching( + /^could not be decrypted: .*wrong secret key/, + ), + }), + "12": expect.objectContaining({ + fileID: 12, + error: expect.stringMatching(/^could not be decrypted: .*JSON/), + }), + }); + expect(stderr.text).toMatch( + /^FAILED album 2: could not be decrypted: .*wrong secret key/m, + ); + expect(stderr.text).toContain(" Failed: 2\n"); + }); + + it("names each album it cannot decrypt in the --json result", async () => { + expect(await backup(brokenAccount(), { json: true })).toBe(1); + + expect(JSON.parse(stdout.text)).toMatchObject({ + downloaded: 1, + failed: 2, + undecryptableAlbums: [2], + }); + }); + + it("still reports them on the next run, though the server does not send them again", async () => { + const ente = brokenAccount(); + await backup(ente); + + // Nothing changed on the server, so this run's refresh gets no rows at + // all. What it reports, it remembered from the first run. + expect(await backup(ente)).toBe(1); + + expect(Object.keys(failures())).toEqual(["11", "12"]); + expect(failures()["11"]!.attempts).toBe(2); + expect(stderr.text).toMatch(/^FAILED album 2: could not be decrypted/m); + }); + + it("drops each once the server sends a copy that decrypts, and then exits 0", async () => { + const ente = brokenAccount(); + await backup(ente); + + // At time 2 files 11 and 12 and album 2 change on the server, and now + // each decrypts. Album 2 brings file 20 with it. + ente.putFile(1, 11, 2); + ente.putFile(1, 12, 2); + ente.putAlbum(2, "Family", 2); + expect(await backup(ente)).toBe(0); + + expect([11, 12, 20].map(stored)).toEqual([true, true, true]); + expect(existsSync(join(backupDir(), "failures.json"))).toBe(false); + expect(stderr.text).not.toContain("FAILED"); + }); + + it("drops each once the server deletes it, and then exits 0", async () => { + const ente = brokenAccount(); + await backup(ente); + + ente.deleteFile(1, 11, 2); + ente.deleteFile(1, 12, 2); + ente.deleteAlbum(2, 2); + expect(await backup(ente)).toBe(0); + + expect(existsSync(join(backupDir(), "failures.json"))).toBe(false); + expect(stderr.text).not.toContain("FAILED"); + }); + + it("backs up a file that does not decrypt in one album but does in another, and does not report it", async () => { + // File 30 is in albums 1 and 3, each holding its key sealed under the + // album's own key. In album 1 that is corrupt; in album 3 it is not, + // and one copy that decrypts is all the backup needs. + const ente = new FakeEnte(); + ente.putAlbum(1, "Trip", 1); + ente.putFile(1, 30, 1, { corruptKey: true }); + ente.putAlbum(3, "Favorites", 1); + ente.putFile(3, 30, 1); + + expect(await backup(ente)).toBe(0); + + expect(stored(30)).toBe(true); + expect(failures()).toEqual({}); + }); +}); + +describe("the read commands with records quak cannot decrypt", () => { + it("quak collections and quak files answer from what decrypts", async () => { + const ente = brokenAccount(); + + expect(await collectionsCommand(context(ente), {})).toBe(0); + expect(stdout.text).toBe("1\talbum\tTrip\n"); + + expect(await filesCommand(context(ente), { collection: "1" })).toBe(0); + expect(stdout.text).toBe("10\timage\tphoto-10.jpg\n"); + }); +}); diff --git a/test/client/enumeration.test.ts b/test/client/enumeration.test.ts index 363f27d..49fcb07 100644 --- a/test/client/enumeration.test.ts +++ b/test/client/enumeration.test.ts @@ -10,7 +10,9 @@ * The return shape keeps live records and tombstones apart — `collections` / * `files` are decrypted live records, `deleted` is a plain list of the ids the * server tombstoned. A tombstone carries no decryptable key or metadata, so it - * is a bare id rather than a hollowed-out `Collection` / `EnteFile`. + * is a bare id rather than a hollowed-out `Collection` / `EnteFile`. A live row + * that does not decrypt, or whose metadata is not JSON, is listed under + * `undecryptable` with the reason, so one bad row does not fail the call. * * All tests inject a fake `fetch` and drive a real `Client` (built with * `Client.fromJSON`) so the decryption path runs for real. Live rows are built @@ -91,27 +93,38 @@ const ownedCollectionRow = ( }; }; -/** A live file row inside a collection, keyed under that collection's key. */ +/** + * A live file row inside a collection, keyed under that collection's key. Its + * metadata decrypts to `metadata` when given, otherwise to the JSON a client + * uploads for `title`. + */ const fileRow = ( collectionKey: Uint8Array, - opts: { id: number; title: string; updationTime: number }, + opts: { + id: number; + title: string; + updationTime: number; + metadata?: string; + }, ): Record => { const fileKey = sodium.crypto_secretbox_keygen(); const { ciphertext: encFileKey, nonce: fileKeyNonce } = secretboxEncrypt( fileKey, collectionKey, ); - const metadata = { - title: opts.title, - fileType: 0, - creationTime: opts.updationTime, - modificationTime: opts.updationTime, - }; + const metadata = + opts.metadata ?? + JSON.stringify({ + title: opts.title, + fileType: 0, + creationTime: opts.updationTime, + modificationTime: opts.updationTime, + }); const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(fileKey); const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push( push.state, - new TextEncoder().encode(JSON.stringify(metadata)), + new TextEncoder().encode(metadata), null, sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, ); @@ -131,6 +144,21 @@ const fileRow = ( }; }; +/** + * `row` with its sealed key replaced by random bytes of the same length, as a + * corrupt row arrives: no key opens it, so the row cannot be decrypted. + */ +const withCorruptKey = ( + row: Record, +): Record => ({ + ...row, + encryptedKey: toBase64( + sodium.randombytes_buf( + sodium.crypto_secretbox_KEYBYTES + sodium.crypto_secretbox_MACBYTES, + ), + ), +}); + /** A tombstone row. Never decrypted, so only these fields are ever read. */ const tombstoneRow = ( id: number, @@ -278,6 +306,58 @@ describe("Client.filesSince", () => { // It gave up after the first page rather than looping. expect(sinceTimes).toEqual(["50"]); }); + + it("leaves out each file it cannot decrypt or parse, with the reason, and decrypts the rest", async () => { + const keys = buildKeys(); + const collectionKey = sodium.crypto_secretbox_keygen(); + + // File 1002's key is sealed under no key quak holds, and file 1003's + // metadata decrypts to text that is not JSON. Neither may cost the + // caller file 1001: each is left out of `files` and reported under + // `undecryptable` with the reason instead of failing the call. + const { fetch } = recordingFetch( + jsonResponse({ + diff: [ + fileRow(collectionKey, { + id: 1001, + title: "fine.jpg", + updationTime: 100, + }), + withCorruptKey( + fileRow(collectionKey, { + id: 1002, + title: "bad-key.jpg", + updationTime: 200, + }), + ), + fileRow(collectionKey, { + id: 1003, + title: "bad-metadata.jpg", + updationTime: 300, + metadata: "not json", + }), + ], + hasMore: false, + }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const { files, undecryptable, cursor } = await client.filesSince({ + collectionID: 1, + collectionKey, + sinceTime: 0, + }); + + expect(files.map((f) => f.id)).toEqual([1001]); + expect(undecryptable).toEqual([ + { id: 1002, error: expect.stringContaining("wrong secret key") }, + { id: 1003, error: expect.stringContaining("JSON") }, + ]); + // The cursor moves past the two left out, as past any other row, so + // the server does not send them again until its copy changes. The + // caller has to keep what it was told here. + expect(cursor).toBe(300); + }); }); describe("Client.collectionsSince", () => { @@ -331,6 +411,42 @@ describe("Client.collectionsSince", () => { expect(result.cursor).toBe(150); expect(sinceTimes).toEqual(["150"]); }); + + it("leaves out an album it cannot decrypt, with the reason, and decrypts the rest", async () => { + const keys = buildKeys(); + + // Album 2's key is sealed under no key quak holds. Without its key + // neither its name nor its files can be read, so it is left out and + // reported under `undecryptable`; album 1 is unaffected. + const { fetch } = recordingFetch( + jsonResponse({ + collections: [ + ownedCollectionRow(keys.masterKey, { + id: 1, + name: "Vacation", + updationTime: 100, + }), + withCorruptKey( + ownedCollectionRow(keys.masterKey, { + id: 2, + name: "Broken", + updationTime: 200, + }), + ), + ], + }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const { collections, undecryptable, cursor } = + await client.collectionsSince({ sinceTime: 0 }); + + expect(collections.map((c) => c.name)).toEqual(["Vacation"]); + expect(undecryptable).toEqual([ + { id: 2, error: expect.stringContaining("wrong secret key") }, + ]); + expect(cursor).toBe(200); + }); }); describe("Client list wrappers still hide deletions", () => {