Resumable, deletion-aware collection and file enumeration (closes #38)

Add collectionsSince/filesSince on Client: they take a starting cursor,
decrypt live records, surface tombstoned ids in a separate `deleted`
list, and return the max updationTime seen as the cursor to resume from.
A tombstone has no key or metadata to decrypt, so it is a bare id rather
than a hollowed-out record that strict tsc would reject.

filesSince paginates the diff from the given cursor and, when the server
reports hasMore but the page's max updationTime does not exceed the
cursor it was fetched with, throws instead of looping forever.

listCollections/listFiles become thin wrappers that enumerate from
sinceTime 0 and drop deletions, so existing callers and tests are
unaffected.

Closes #7.

Model: opus-4-8

TODO.md

@@ -18,6 +18,15 @@ Update the README API reference section to match the current implementation.
 
 # Completed Steps
 
+- 2026-09-22: Added resumable, deletion-aware enumeration to `Client` (issue 38,
+  closes issue 7). `collectionsSince`/`filesSince` take a starting cursor,
+  decrypt live records, surface tombstoned ids in a separate `deleted` list (a
+  tombstone has nothing to decrypt, so it is a bare id, not a hollow record),
+  and return the max `updationTime` seen as the cursor to resume from.
+  `filesSince` refuses to loop when the diff reports `hasMore` without advancing
+  the cursor (issue 7). `listCollections`/`listFiles` are now thin wrappers that
+  enumerate from `sinceTime: 0` and drop deletions, so existing callers are
+  unaffected.
 - 2026-09-22: Carried file size, thumbnail size, and the deletion flag through
   `decryptFile` (issue 37, foundation for the cache/API design). Live files now
   populate `file.size`/`thumbnail.size` from the server's `info` (left

src/client.ts

@@ -37,6 +37,22 @@ export interface ClientSnapshot {
     publicKey: string;
 }
 
+// The result of a resumable enumeration. Live decrypted records and deleted
+// ids are kept apart on purpose: a tombstone carries no key or metadata to
+// decrypt, so it is a bare id rather than a hollowed-out record. `cursor` is
+// the max `updationTime` seen, to pass back into the next call.
+export interface CollectionsPage {
+    collections: Collection[];
+    deleted: number[];
+    cursor: number;
+}
+
+export interface FilesPage {
+    files: EnteFile[];
+    deleted: number[];
+    cursor: number;
+}
+
 export class Client {
     private readonly api: ApiClient;
     private readonly email: string;
@@ -150,52 +166,111 @@ export class Client {
         this.api.clearAuthToken();
     }
 
-    async listCollections(): Promise<Collection[]> {
+    // Enumerate collections changed since `sinceTime`. Live collections are
+    // decrypted; tombstoned ones (isDeleted) are surfaced as bare ids. The
+    // returned cursor is the max `updationTime` seen — including tombstones, so
+    // the next sync resumes past them — falling back to `sinceTime` when the
+    // response is empty. `/collections/v2` returns the whole changed set in one
+    // response, so there is no pagination here.
+    async collectionsSince(args: {
+        sinceTime: number;
+    }): Promise<CollectionsPage> {
         this.assertLoggedIn();
-        const { collections } = await this.api.getJSON<{
+        const { collections: raws } = await this.api.getJSON<{
             collections: RawCollection[];
-        }>("/collections/v2", { sinceTime: 0 });
-        // The sync API keeps returning deleted collections as tombstones
-        // (isDeleted: true); their diff endpoint 404s, so drop them.
-        return collections
-            .filter((raw) => !raw.isDeleted)
-            .map((raw) =>
-                decryptCollection(
-                    raw,
-                    {
-                        masterKey: this.masterKey,
-                        publicKey: this.publicKey,
-                        secretKey: this.secretKey,
-                    },
-                    this.userID,
-                ),
-            );
+        }>("/collections/v2", { sinceTime: args.sinceTime });
+
+        const collections: Collection[] = [];
+        const deleted: number[] = [];
+        let cursor = args.sinceTime;
+        for (const raw of raws) {
+            if (raw.isDeleted) {
+                deleted.push(raw.id);
+            } else {
+                collections.push(
+                    decryptCollection(
+                        raw,
+                        {
+                            masterKey: this.masterKey,
+                            publicKey: this.publicKey,
+                            secretKey: this.secretKey,
+                        },
+                        this.userID,
+                    ),
+                );
+            }
+            if (raw.updationTime > cursor) cursor = raw.updationTime;
+        }
+        return { collections, deleted, cursor };
     }
 
-    async listFiles(
-        collectionID: number,
-        collectionKey: Uint8Array,
-    ): Promise<EnteFile[]> {
+    // Enumerate a collection's files changed since `sinceTime`, paginating the
+    // diff from that cursor. Live rows are decrypted; tombstoned ones are
+    // surfaced as bare ids. Returns the final cursor to resume from.
+    async filesSince(args: {
+        collectionID: number;
+        collectionKey: Uint8Array;
+        sinceTime: number;
+    }): Promise<FilesPage> {
         this.assertLoggedIn();
-        const allFiles: EnteFile[] = [];
-        let sinceTime = 0;
+        const { collectionID, collectionKey } = args;
+        const files: EnteFile[] = [];
+        const deleted: number[] = [];
+        let cursor = args.sinceTime;
         for (;;) {
             const { diff, hasMore } = await this.api.getJSON<{
                 diff: RawEnteFile[];
                 hasMore: boolean;
-            }>("/collections/v2/diff", { collectionID, sinceTime });
+            }>("/collections/v2/diff", { collectionID, sinceTime: cursor });
 
+            let pageMax = cursor;
             for (const raw of diff) {
-                if (!raw.isDeleted) {
-                    allFiles.push(decryptFile(raw, collectionKey));
-                }
-                if (raw.updationTime > sinceTime) {
-                    sinceTime = raw.updationTime;
+                if (raw.isDeleted) {
+                    deleted.push(raw.id);
+                } else {
+                    files.push(decryptFile(raw, collectionKey));
                 }
+                if (raw.updationTime > pageMax) pageMax = raw.updationTime;
             }
-            if (!hasMore) break;
+
+            if (!hasMore) {
+                cursor = pageMax;
+                break;
+            }
+            // The server says there is more, but this page did not advance the
+            // cursor: following hasMore would refetch the same page forever
+            // (#7). Stop with a clear error instead of looping.
+            if (pageMax <= cursor) {
+                throw new Error(
+                    `/collections/v2/diff for collection ${collectionID} ` +
+                        `returned hasMore with a cursor that did not advance ` +
+                        `(stuck at ${cursor}); refusing to loop`,
+                );
+            }
+            cursor = pageMax;
         }
-        return allFiles;
+        return { files, deleted, cursor };
+    }
+
+    // Whole-account listing: every live collection, deletions hidden. A thin
+    // wrapper over `collectionsSince` from the beginning of time.
+    async listCollections(): Promise<Collection[]> {
+        const { collections } = await this.collectionsSince({ sinceTime: 0 });
+        return collections;
+    }
+
+    // Every live file in a collection, deletions hidden. A thin wrapper over
+    // `filesSince` from the beginning of time.
+    async listFiles(
+        collectionID: number,
+        collectionKey: Uint8Array,
+    ): Promise<EnteFile[]> {
+        const { files } = await this.filesSince({
+            collectionID,
+            collectionKey,
+            sinceTime: 0,
+        });
+        return files;
     }
 
     async downloadFile(

src/index.ts

@@ -1,6 +1,12 @@
 export const VERSION = "0.0.0";
 
-export { Client, type LoginOptions, type ClientSnapshot } from "./client.js";
+export {
+    Client,
+    type LoginOptions,
+    type ClientSnapshot,
+    type CollectionsPage,
+    type FilesPage,
+} from "./client.js";
 export {
     ApiClient,
     ApiError,

test/client/enumeration.test.ts

@@ -0,0 +1,367 @@
+/**
+ * Tests for the resumable, deletion-aware enumeration variants on `Client`:
+ * `collectionsSince` and `filesSince`.
+ *
+ * The whole-account methods `listCollections` / `listFiles` always start at
+ * `sinceTime: 0` and hide deletions. The cache refresh needs the opposite:
+ * start from a saved cursor, learn what was deleted, and get back a cursor to
+ * resume from next time. These two methods provide that.
+ *
+ * The return shape keeps live records and tombstones apart — `collections` /
+ * `files` are decrypted live records, `deleted` is a plain list of the ids the
+ * server tombstoned. A tombstone carries no decryptable key or metadata, so it
+ * is a bare id rather than a hollowed-out `Collection` / `EnteFile`.
+ *
+ * All tests inject a fake `fetch` and drive a real `Client` (built with
+ * `Client.fromJSON`) so the decryption path runs for real. Live rows are built
+ * with libsodium exactly as the server would encrypt them; tombstone rows carry
+ * only the fields the code reads (`id`, `updationTime`, `isDeleted`), because
+ * they are never decrypted.
+ */
+
+import sodium from "libsodium-wrappers-sumo";
+import { beforeAll, describe, expect, it } from "vitest";
+import { init, toBase64 } from "../../src/crypto/index.js";
+import { Client, type ClientSnapshot } from "../../src/client.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+const USER_ID = 42;
+
+interface Keys {
+    masterKey: Uint8Array;
+    publicKey: Uint8Array;
+    secretKey: Uint8Array;
+}
+
+const buildKeys = (): Keys => {
+    const kp = sodium.crypto_box_keypair();
+    return {
+        masterKey: sodium.crypto_secretbox_keygen(),
+        publicKey: kp.publicKey,
+        secretKey: kp.privateKey,
+    };
+};
+
+const snapshotFor = (keys: Keys): ClientSnapshot => ({
+    email: "user@example.com",
+    userID: USER_ID,
+    token: "test-token",
+    masterKey: toBase64(keys.masterKey),
+    secretKey: toBase64(keys.secretKey),
+    publicKey: toBase64(keys.publicKey),
+});
+
+const secretboxEncrypt = (
+    plaintext: Uint8Array,
+    key: Uint8Array,
+): { ciphertext: Uint8Array; nonce: Uint8Array } => {
+    const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
+    return {
+        ciphertext: sodium.crypto_secretbox_easy(plaintext, nonce, key),
+        nonce,
+    };
+};
+
+/** An owned collection row as the server sends it, keyed under the master key. */
+const ownedCollectionRow = (
+    masterKey: Uint8Array,
+    opts: { id: number; name: string; updationTime: number },
+): Record<string, unknown> => {
+    const collectionKey = sodium.crypto_secretbox_keygen();
+    const { ciphertext: encKey, nonce: keyNonce } = secretboxEncrypt(
+        collectionKey,
+        masterKey,
+    );
+    const { ciphertext: encName, nonce: nameNonce } = secretboxEncrypt(
+        new TextEncoder().encode(opts.name),
+        collectionKey,
+    );
+    return {
+        id: opts.id,
+        owner: { id: USER_ID },
+        encryptedKey: toBase64(encKey),
+        keyDecryptionNonce: toBase64(keyNonce),
+        encryptedName: toBase64(encName),
+        nameDecryptionNonce: toBase64(nameNonce),
+        type: "album",
+        updationTime: opts.updationTime,
+    };
+};
+
+/** A live file row inside a collection, keyed under that collection's key. */
+const fileRow = (
+    collectionKey: Uint8Array,
+    opts: { id: number; title: string; updationTime: number },
+): Record<string, unknown> => {
+    const fileKey = sodium.crypto_secretbox_keygen();
+    const { ciphertext: encFileKey, nonce: fileKeyNonce } = secretboxEncrypt(
+        fileKey,
+        collectionKey,
+    );
+    const metadata = {
+        title: opts.title,
+        fileType: 0,
+        creationTime: opts.updationTime,
+        modificationTime: opts.updationTime,
+    };
+    const push =
+        sodium.crypto_secretstream_xchacha20poly1305_init_push(fileKey);
+    const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push(
+        push.state,
+        new TextEncoder().encode(JSON.stringify(metadata)),
+        null,
+        sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
+    );
+    return {
+        id: opts.id,
+        collectionID: 1,
+        ownerID: USER_ID,
+        encryptedKey: toBase64(encFileKey),
+        keyDecryptionNonce: toBase64(fileKeyNonce),
+        metadata: {
+            encryptedData: toBase64(encMeta),
+            decryptionHeader: toBase64(push.header),
+        },
+        file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
+        thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
+        updationTime: opts.updationTime,
+    };
+};
+
+/** A tombstone row. Never decrypted, so only these fields are ever read. */
+const tombstoneRow = (
+    id: number,
+    updationTime: number,
+): Record<string, unknown> => ({
+    id,
+    updationTime,
+    isDeleted: true,
+});
+
+const jsonResponse = (body: unknown): Response =>
+    new Response(JSON.stringify(body), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+    });
+
+/**
+ * A fetch that serves canned responses in order and records the `sinceTime`
+ * query parameter each request carried, so tests can prove the cursor is
+ * threaded from one page (and one call) to the next.
+ */
+const recordingFetch = (
+    ...responses: Response[]
+): { fetch: typeof globalThis.fetch; sinceTimes: (string | null)[] } => {
+    const sinceTimes: (string | null)[] = [];
+    let i = 0;
+    const fake = async (input: RequestInfo | URL): Promise<Response> => {
+        const url =
+            typeof input === "string"
+                ? input
+                : input instanceof URL
+                  ? input.href
+                  : input.url;
+        sinceTimes.push(new URL(url).searchParams.get("sinceTime"));
+        if (i >= responses.length) {
+            throw new Error(`recordingFetch: no response for call #${i}`);
+        }
+        return responses[i++]!;
+    };
+    return { fetch: fake as typeof globalThis.fetch, sinceTimes };
+};
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("Client.filesSince", () => {
+    beforeAll(async () => {
+        await init();
+        await sodium.ready;
+    });
+
+    it("pages from the given cursor, decrypts live rows, and collects tombstones", async () => {
+        const keys = buildKeys();
+        const collectionKey = sodium.crypto_secretbox_keygen();
+
+        // Page 1 mixes a live file and a tombstone; the tombstone has the
+        // higher updationTime, so it — not the live row — sets the cursor the
+        // second page must be fetched from.
+        const { fetch, sinceTimes } = recordingFetch(
+            jsonResponse({
+                diff: [
+                    fileRow(collectionKey, {
+                        id: 1001,
+                        title: "first.jpg",
+                        updationTime: 100,
+                    }),
+                    tombstoneRow(1002, 150),
+                ],
+                hasMore: true,
+            }),
+            jsonResponse({
+                diff: [
+                    fileRow(collectionKey, {
+                        id: 1003,
+                        title: "second.jpg",
+                        updationTime: 200,
+                    }),
+                ],
+                hasMore: false,
+            }),
+        );
+        const client = Client.fromJSON(snapshotFor(keys), { fetch });
+
+        const { files, deleted, cursor } = await client.filesSince({
+            collectionID: 1,
+            collectionKey,
+            sinceTime: 0,
+        });
+
+        expect(files.map((f) => f.id)).toEqual([1001, 1003]);
+        expect(files.map((f) => f.metadata.title)).toEqual([
+            "first.jpg",
+            "second.jpg",
+        ]);
+        expect(deleted).toEqual([1002]);
+        expect(cursor).toBe(200);
+        // First request started at the caller's cursor; the second resumed
+        // from the max updationTime seen on the first page (the tombstone's).
+        expect(sinceTimes).toEqual(["0", "150"]);
+    });
+
+    it("fetches only newer rows when the returned cursor is passed back in", async () => {
+        const keys = buildKeys();
+        const collectionKey = sodium.crypto_secretbox_keygen();
+
+        const { fetch, sinceTimes } = recordingFetch(
+            jsonResponse({ diff: [], hasMore: false }),
+        );
+        const client = Client.fromJSON(snapshotFor(keys), { fetch });
+
+        const result = await client.filesSince({
+            collectionID: 1,
+            collectionKey,
+            sinceTime: 200,
+        });
+
+        expect(result.files).toEqual([]);
+        expect(result.deleted).toEqual([]);
+        // An empty diff advances nothing: the cursor falls back to the input.
+        expect(result.cursor).toBe(200);
+        expect(sinceTimes).toEqual(["200"]);
+    });
+
+    it("stops and throws when the server claims more but does not advance (#7)", async () => {
+        const keys = buildKeys();
+        const collectionKey = sodium.crypto_secretbox_keygen();
+
+        // hasMore is true, but the page's max updationTime (50) does not exceed
+        // the cursor the request was made with (50). Following hasMore here
+        // would refetch this same page forever.
+        const { fetch, sinceTimes } = recordingFetch(
+            jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }),
+            jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }),
+        );
+        const client = Client.fromJSON(snapshotFor(keys), { fetch });
+
+        await expect(
+            client.filesSince({
+                collectionID: 1,
+                collectionKey,
+                sinceTime: 50,
+            }),
+        ).rejects.toThrow(/not advance|non-advancing/i);
+        // It gave up after the first page rather than looping.
+        expect(sinceTimes).toEqual(["50"]);
+    });
+});
+
+describe("Client.collectionsSince", () => {
+    beforeAll(async () => {
+        await init();
+        await sodium.ready;
+    });
+
+    it("decrypts live collections, collects tombstones, and returns a cursor", async () => {
+        const keys = buildKeys();
+
+        const { fetch, sinceTimes } = recordingFetch(
+            jsonResponse({
+                collections: [
+                    ownedCollectionRow(keys.masterKey, {
+                        id: 1,
+                        name: "Vacation",
+                        updationTime: 100,
+                    }),
+                    tombstoneRow(3, 150),
+                ],
+            }),
+        );
+        const client = Client.fromJSON(snapshotFor(keys), { fetch });
+
+        const { collections, deleted, cursor } = await client.collectionsSince({
+            sinceTime: 0,
+        });
+
+        expect(collections.map((c) => c.id)).toEqual([1]);
+        expect(collections[0]!.name).toBe("Vacation");
+        expect(deleted).toEqual([3]);
+        // The tombstone's updationTime advances the cursor too, so the next
+        // sync starts after it rather than seeing it again.
+        expect(cursor).toBe(150);
+        expect(sinceTimes).toEqual(["0"]);
+    });
+
+    it("falls back to the input cursor on an empty response", async () => {
+        const keys = buildKeys();
+
+        const { fetch, sinceTimes } = recordingFetch(
+            jsonResponse({ collections: [] }),
+        );
+        const client = Client.fromJSON(snapshotFor(keys), { fetch });
+
+        const result = await client.collectionsSince({ sinceTime: 150 });
+
+        expect(result.collections).toEqual([]);
+        expect(result.deleted).toEqual([]);
+        expect(result.cursor).toBe(150);
+        expect(sinceTimes).toEqual(["150"]);
+    });
+});
+
+describe("Client list wrappers still hide deletions", () => {
+    beforeAll(async () => {
+        await init();
+        await sodium.ready;
+    });
+
+    it("listFiles drops tombstones and returns only live files", async () => {
+        const keys = buildKeys();
+        const collectionKey = sodium.crypto_secretbox_keygen();
+
+        const { fetch, sinceTimes } = recordingFetch(
+            jsonResponse({
+                diff: [
+                    fileRow(collectionKey, {
+                        id: 7,
+                        title: "keep.jpg",
+                        updationTime: 100,
+                    }),
+                    tombstoneRow(8, 150),
+                ],
+                hasMore: false,
+            }),
+        );
+        const client = Client.fromJSON(snapshotFor(keys), { fetch });
+
+        const files = await client.listFiles(1, collectionKey);
+
+        expect(files.map((f) => f.id)).toEqual([7]);
+        // The wrapper starts a full enumeration from zero.
+        expect(sinceTimes).toEqual(["0"]);
+    });
+});