diff --git a/TODO.md b/TODO.md index ccd4e66..faa3564 100644 --- a/TODO.md +++ b/TODO.md @@ -18,6 +18,15 @@ Update the README API reference section to match the current implementation. # Completed Steps +- 2026-09-22: Added resumable, deletion-aware enumeration to `Client` (issue 38, + closes issue 7). `collectionsSince`/`filesSince` take a starting cursor, + decrypt live records, surface tombstoned ids in a separate `deleted` list (a + tombstone has nothing to decrypt, so it is a bare id, not a hollow record), + and return the max `updationTime` seen as the cursor to resume from. + `filesSince` refuses to loop when the diff reports `hasMore` without advancing + the cursor (issue 7). `listCollections`/`listFiles` are now thin wrappers that + enumerate from `sinceTime: 0` and drop deletions, so existing callers are + unaffected. - 2026-09-22: Carried file size, thumbnail size, and the deletion flag through `decryptFile` (issue 37, foundation for the cache/API design). Live files now populate `file.size`/`thumbnail.size` from the server's `info` (left diff --git a/src/client.ts b/src/client.ts index a4d14ce..0d3fabe 100644 --- a/src/client.ts +++ b/src/client.ts @@ -37,6 +37,22 @@ export interface ClientSnapshot { publicKey: string; } +// The result of a resumable enumeration. Live decrypted records and deleted +// ids are kept apart on purpose: a tombstone carries no key or metadata to +// decrypt, so it is a bare id rather than a hollowed-out record. `cursor` is +// the max `updationTime` seen, to pass back into the next call. +export interface CollectionsPage { + collections: Collection[]; + deleted: number[]; + cursor: number; +} + +export interface FilesPage { + files: EnteFile[]; + deleted: number[]; + cursor: number; +} + export class Client { private readonly api: ApiClient; private readonly email: string; @@ -150,52 +166,111 @@ export class Client { this.api.clearAuthToken(); } - async listCollections(): Promise { + // Enumerate collections changed since `sinceTime`. Live collections are + // decrypted; tombstoned ones (isDeleted) are surfaced as bare ids. The + // returned cursor is the max `updationTime` seen — including tombstones, so + // the next sync resumes past them — falling back to `sinceTime` when the + // response is empty. `/collections/v2` returns the whole changed set in one + // response, so there is no pagination here. + async collectionsSince(args: { + sinceTime: number; + }): Promise { this.assertLoggedIn(); - const { collections } = await this.api.getJSON<{ + const { collections: raws } = await this.api.getJSON<{ collections: RawCollection[]; - }>("/collections/v2", { sinceTime: 0 }); - // The sync API keeps returning deleted collections as tombstones - // (isDeleted: true); their diff endpoint 404s, so drop them. - return collections - .filter((raw) => !raw.isDeleted) - .map((raw) => - decryptCollection( - raw, - { - masterKey: this.masterKey, - publicKey: this.publicKey, - secretKey: this.secretKey, - }, - this.userID, - ), - ); + }>("/collections/v2", { sinceTime: args.sinceTime }); + + const collections: Collection[] = []; + const deleted: number[] = []; + let cursor = args.sinceTime; + for (const raw of raws) { + if (raw.isDeleted) { + deleted.push(raw.id); + } else { + collections.push( + decryptCollection( + raw, + { + masterKey: this.masterKey, + publicKey: this.publicKey, + secretKey: this.secretKey, + }, + this.userID, + ), + ); + } + if (raw.updationTime > cursor) cursor = raw.updationTime; + } + return { collections, deleted, cursor }; } - async listFiles( - collectionID: number, - collectionKey: Uint8Array, - ): Promise { + // Enumerate a collection's files changed since `sinceTime`, paginating the + // diff from that cursor. Live rows are decrypted; tombstoned ones are + // surfaced as bare ids. Returns the final cursor to resume from. + async filesSince(args: { + collectionID: number; + collectionKey: Uint8Array; + sinceTime: number; + }): Promise { this.assertLoggedIn(); - const allFiles: EnteFile[] = []; - let sinceTime = 0; + const { collectionID, collectionKey } = args; + const files: EnteFile[] = []; + const deleted: number[] = []; + let cursor = args.sinceTime; for (;;) { const { diff, hasMore } = await this.api.getJSON<{ diff: RawEnteFile[]; hasMore: boolean; - }>("/collections/v2/diff", { collectionID, sinceTime }); + }>("/collections/v2/diff", { collectionID, sinceTime: cursor }); + let pageMax = cursor; for (const raw of diff) { - if (!raw.isDeleted) { - allFiles.push(decryptFile(raw, collectionKey)); - } - if (raw.updationTime > sinceTime) { - sinceTime = raw.updationTime; + if (raw.isDeleted) { + deleted.push(raw.id); + } else { + files.push(decryptFile(raw, collectionKey)); } + if (raw.updationTime > pageMax) pageMax = raw.updationTime; } - if (!hasMore) break; + + if (!hasMore) { + cursor = pageMax; + break; + } + // The server says there is more, but this page did not advance the + // cursor: following hasMore would refetch the same page forever + // (#7). Stop with a clear error instead of looping. + if (pageMax <= cursor) { + throw new Error( + `/collections/v2/diff for collection ${collectionID} ` + + `returned hasMore with a cursor that did not advance ` + + `(stuck at ${cursor}); refusing to loop`, + ); + } + cursor = pageMax; } - return allFiles; + return { files, deleted, cursor }; + } + + // Whole-account listing: every live collection, deletions hidden. A thin + // wrapper over `collectionsSince` from the beginning of time. + async listCollections(): Promise { + const { collections } = await this.collectionsSince({ sinceTime: 0 }); + return collections; + } + + // Every live file in a collection, deletions hidden. A thin wrapper over + // `filesSince` from the beginning of time. + async listFiles( + collectionID: number, + collectionKey: Uint8Array, + ): Promise { + const { files } = await this.filesSince({ + collectionID, + collectionKey, + sinceTime: 0, + }); + return files; } async downloadFile( diff --git a/src/index.ts b/src/index.ts index c53ceb9..f9e34c2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,12 @@ export const VERSION = "0.0.0"; -export { Client, type LoginOptions, type ClientSnapshot } from "./client.js"; +export { + Client, + type LoginOptions, + type ClientSnapshot, + type CollectionsPage, + type FilesPage, +} from "./client.js"; export { ApiClient, ApiError, diff --git a/test/client/enumeration.test.ts b/test/client/enumeration.test.ts new file mode 100644 index 0000000..363f27d --- /dev/null +++ b/test/client/enumeration.test.ts @@ -0,0 +1,367 @@ +/** + * Tests for the resumable, deletion-aware enumeration variants on `Client`: + * `collectionsSince` and `filesSince`. + * + * The whole-account methods `listCollections` / `listFiles` always start at + * `sinceTime: 0` and hide deletions. The cache refresh needs the opposite: + * start from a saved cursor, learn what was deleted, and get back a cursor to + * resume from next time. These two methods provide that. + * + * The return shape keeps live records and tombstones apart — `collections` / + * `files` are decrypted live records, `deleted` is a plain list of the ids the + * server tombstoned. A tombstone carries no decryptable key or metadata, so it + * is a bare id rather than a hollowed-out `Collection` / `EnteFile`. + * + * All tests inject a fake `fetch` and drive a real `Client` (built with + * `Client.fromJSON`) so the decryption path runs for real. Live rows are built + * with libsodium exactly as the server would encrypt them; tombstone rows carry + * only the fields the code reads (`id`, `updationTime`, `isDeleted`), because + * they are never decrypted. + */ + +import sodium from "libsodium-wrappers-sumo"; +import { beforeAll, describe, expect, it } from "vitest"; +import { init, toBase64 } from "../../src/crypto/index.js"; +import { Client, type ClientSnapshot } from "../../src/client.js"; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const USER_ID = 42; + +interface Keys { + masterKey: Uint8Array; + publicKey: Uint8Array; + secretKey: Uint8Array; +} + +const buildKeys = (): Keys => { + const kp = sodium.crypto_box_keypair(); + return { + masterKey: sodium.crypto_secretbox_keygen(), + publicKey: kp.publicKey, + secretKey: kp.privateKey, + }; +}; + +const snapshotFor = (keys: Keys): ClientSnapshot => ({ + email: "user@example.com", + userID: USER_ID, + token: "test-token", + masterKey: toBase64(keys.masterKey), + secretKey: toBase64(keys.secretKey), + publicKey: toBase64(keys.publicKey), +}); + +const secretboxEncrypt = ( + plaintext: Uint8Array, + key: Uint8Array, +): { ciphertext: Uint8Array; nonce: Uint8Array } => { + const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + return { + ciphertext: sodium.crypto_secretbox_easy(plaintext, nonce, key), + nonce, + }; +}; + +/** An owned collection row as the server sends it, keyed under the master key. */ +const ownedCollectionRow = ( + masterKey: Uint8Array, + opts: { id: number; name: string; updationTime: number }, +): Record => { + const collectionKey = sodium.crypto_secretbox_keygen(); + const { ciphertext: encKey, nonce: keyNonce } = secretboxEncrypt( + collectionKey, + masterKey, + ); + const { ciphertext: encName, nonce: nameNonce } = secretboxEncrypt( + new TextEncoder().encode(opts.name), + collectionKey, + ); + return { + id: opts.id, + owner: { id: USER_ID }, + encryptedKey: toBase64(encKey), + keyDecryptionNonce: toBase64(keyNonce), + encryptedName: toBase64(encName), + nameDecryptionNonce: toBase64(nameNonce), + type: "album", + updationTime: opts.updationTime, + }; +}; + +/** A live file row inside a collection, keyed under that collection's key. */ +const fileRow = ( + collectionKey: Uint8Array, + opts: { id: number; title: string; updationTime: number }, +): Record => { + const fileKey = sodium.crypto_secretbox_keygen(); + const { ciphertext: encFileKey, nonce: fileKeyNonce } = secretboxEncrypt( + fileKey, + collectionKey, + ); + const metadata = { + title: opts.title, + fileType: 0, + creationTime: opts.updationTime, + modificationTime: opts.updationTime, + }; + const push = + sodium.crypto_secretstream_xchacha20poly1305_init_push(fileKey); + const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push( + push.state, + new TextEncoder().encode(JSON.stringify(metadata)), + null, + sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, + ); + return { + id: opts.id, + collectionID: 1, + ownerID: USER_ID, + encryptedKey: toBase64(encFileKey), + keyDecryptionNonce: toBase64(fileKeyNonce), + metadata: { + encryptedData: toBase64(encMeta), + decryptionHeader: toBase64(push.header), + }, + file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, + thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, + updationTime: opts.updationTime, + }; +}; + +/** A tombstone row. Never decrypted, so only these fields are ever read. */ +const tombstoneRow = ( + id: number, + updationTime: number, +): Record => ({ + id, + updationTime, + isDeleted: true, +}); + +const jsonResponse = (body: unknown): Response => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); + +/** + * A fetch that serves canned responses in order and records the `sinceTime` + * query parameter each request carried, so tests can prove the cursor is + * threaded from one page (and one call) to the next. + */ +const recordingFetch = ( + ...responses: Response[] +): { fetch: typeof globalThis.fetch; sinceTimes: (string | null)[] } => { + const sinceTimes: (string | null)[] = []; + let i = 0; + const fake = async (input: RequestInfo | URL): Promise => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.href + : input.url; + sinceTimes.push(new URL(url).searchParams.get("sinceTime")); + if (i >= responses.length) { + throw new Error(`recordingFetch: no response for call #${i}`); + } + return responses[i++]!; + }; + return { fetch: fake as typeof globalThis.fetch, sinceTimes }; +}; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("Client.filesSince", () => { + beforeAll(async () => { + await init(); + await sodium.ready; + }); + + it("pages from the given cursor, decrypts live rows, and collects tombstones", async () => { + const keys = buildKeys(); + const collectionKey = sodium.crypto_secretbox_keygen(); + + // Page 1 mixes a live file and a tombstone; the tombstone has the + // higher updationTime, so it — not the live row — sets the cursor the + // second page must be fetched from. + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ + diff: [ + fileRow(collectionKey, { + id: 1001, + title: "first.jpg", + updationTime: 100, + }), + tombstoneRow(1002, 150), + ], + hasMore: true, + }), + jsonResponse({ + diff: [ + fileRow(collectionKey, { + id: 1003, + title: "second.jpg", + updationTime: 200, + }), + ], + hasMore: false, + }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const { files, deleted, cursor } = await client.filesSince({ + collectionID: 1, + collectionKey, + sinceTime: 0, + }); + + expect(files.map((f) => f.id)).toEqual([1001, 1003]); + expect(files.map((f) => f.metadata.title)).toEqual([ + "first.jpg", + "second.jpg", + ]); + expect(deleted).toEqual([1002]); + expect(cursor).toBe(200); + // First request started at the caller's cursor; the second resumed + // from the max updationTime seen on the first page (the tombstone's). + expect(sinceTimes).toEqual(["0", "150"]); + }); + + it("fetches only newer rows when the returned cursor is passed back in", async () => { + const keys = buildKeys(); + const collectionKey = sodium.crypto_secretbox_keygen(); + + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ diff: [], hasMore: false }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const result = await client.filesSince({ + collectionID: 1, + collectionKey, + sinceTime: 200, + }); + + expect(result.files).toEqual([]); + expect(result.deleted).toEqual([]); + // An empty diff advances nothing: the cursor falls back to the input. + expect(result.cursor).toBe(200); + expect(sinceTimes).toEqual(["200"]); + }); + + it("stops and throws when the server claims more but does not advance (#7)", async () => { + const keys = buildKeys(); + const collectionKey = sodium.crypto_secretbox_keygen(); + + // hasMore is true, but the page's max updationTime (50) does not exceed + // the cursor the request was made with (50). Following hasMore here + // would refetch this same page forever. + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }), + jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + await expect( + client.filesSince({ + collectionID: 1, + collectionKey, + sinceTime: 50, + }), + ).rejects.toThrow(/not advance|non-advancing/i); + // It gave up after the first page rather than looping. + expect(sinceTimes).toEqual(["50"]); + }); +}); + +describe("Client.collectionsSince", () => { + beforeAll(async () => { + await init(); + await sodium.ready; + }); + + it("decrypts live collections, collects tombstones, and returns a cursor", async () => { + const keys = buildKeys(); + + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ + collections: [ + ownedCollectionRow(keys.masterKey, { + id: 1, + name: "Vacation", + updationTime: 100, + }), + tombstoneRow(3, 150), + ], + }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const { collections, deleted, cursor } = await client.collectionsSince({ + sinceTime: 0, + }); + + expect(collections.map((c) => c.id)).toEqual([1]); + expect(collections[0]!.name).toBe("Vacation"); + expect(deleted).toEqual([3]); + // The tombstone's updationTime advances the cursor too, so the next + // sync starts after it rather than seeing it again. + expect(cursor).toBe(150); + expect(sinceTimes).toEqual(["0"]); + }); + + it("falls back to the input cursor on an empty response", async () => { + const keys = buildKeys(); + + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ collections: [] }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const result = await client.collectionsSince({ sinceTime: 150 }); + + expect(result.collections).toEqual([]); + expect(result.deleted).toEqual([]); + expect(result.cursor).toBe(150); + expect(sinceTimes).toEqual(["150"]); + }); +}); + +describe("Client list wrappers still hide deletions", () => { + beforeAll(async () => { + await init(); + await sodium.ready; + }); + + it("listFiles drops tombstones and returns only live files", async () => { + const keys = buildKeys(); + const collectionKey = sodium.crypto_secretbox_keygen(); + + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ + diff: [ + fileRow(collectionKey, { + id: 7, + title: "keep.jpg", + updationTime: 100, + }), + tombstoneRow(8, 150), + ], + hasMore: false, + }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const files = await client.listFiles(1, collectionKey); + + expect(files.map((f) => f.id)).toEqual([7]); + // The wrapper starts a full enumeration from zero. + expect(sinceTimes).toEqual(["0"]); + }); +});