Resumable, deletion-aware collection and file enumeration (closes #38)

Add collectionsSince/filesSince on Client: they take a starting cursor,
decrypt live records, surface tombstoned ids in a separate `deleted`
list, and return the max updationTime seen as the cursor to resume from.
A tombstone has no key or metadata to decrypt, so it is a bare id rather
than a hollowed-out record that strict tsc would reject.

filesSince paginates the diff from the given cursor and, when the server
reports hasMore but the page's max updationTime does not exceed the
cursor it was fetched with, throws instead of looping forever.

listCollections/listFiles become thin wrappers that enumerate from
sinceTime 0 and drop deletions, so existing callers and tests are
unaffected.

Closes #7.

Model: opus-4-8

src/client.ts

@@ -37,6 +37,22 @@ export interface ClientSnapshot {
     publicKey: string;
 }
 
+// The result of a resumable enumeration. Live decrypted records and deleted
+// ids are kept apart on purpose: a tombstone carries no key or metadata to
+// decrypt, so it is a bare id rather than a hollowed-out record. `cursor` is
+// the max `updationTime` seen, to pass back into the next call.
+export interface CollectionsPage {
+    collections: Collection[];
+    deleted: number[];
+    cursor: number;
+}
+
+export interface FilesPage {
+    files: EnteFile[];
+    deleted: number[];
+    cursor: number;
+}
+
 export class Client {
     private readonly api: ApiClient;
     private readonly email: string;
@@ -150,52 +166,111 @@ export class Client {
         this.api.clearAuthToken();
     }
 
-    async listCollections(): Promise<Collection[]> {
+    // Enumerate collections changed since `sinceTime`. Live collections are
+    // decrypted; tombstoned ones (isDeleted) are surfaced as bare ids. The
+    // returned cursor is the max `updationTime` seen — including tombstones, so
+    // the next sync resumes past them — falling back to `sinceTime` when the
+    // response is empty. `/collections/v2` returns the whole changed set in one
+    // response, so there is no pagination here.
+    async collectionsSince(args: {
+        sinceTime: number;
+    }): Promise<CollectionsPage> {
         this.assertLoggedIn();
-        const { collections } = await this.api.getJSON<{
+        const { collections: raws } = await this.api.getJSON<{
             collections: RawCollection[];
-        }>("/collections/v2", { sinceTime: 0 });
-        // The sync API keeps returning deleted collections as tombstones
-        // (isDeleted: true); their diff endpoint 404s, so drop them.
-        return collections
-            .filter((raw) => !raw.isDeleted)
-            .map((raw) =>
-                decryptCollection(
-                    raw,
-                    {
-                        masterKey: this.masterKey,
-                        publicKey: this.publicKey,
-                        secretKey: this.secretKey,
-                    },
-                    this.userID,
-                ),
-            );
+        }>("/collections/v2", { sinceTime: args.sinceTime });
+
+        const collections: Collection[] = [];
+        const deleted: number[] = [];
+        let cursor = args.sinceTime;
+        for (const raw of raws) {
+            if (raw.isDeleted) {
+                deleted.push(raw.id);
+            } else {
+                collections.push(
+                    decryptCollection(
+                        raw,
+                        {
+                            masterKey: this.masterKey,
+                            publicKey: this.publicKey,
+                            secretKey: this.secretKey,
+                        },
+                        this.userID,
+                    ),
+                );
+            }
+            if (raw.updationTime > cursor) cursor = raw.updationTime;
+        }
+        return { collections, deleted, cursor };
     }
 
-    async listFiles(
-        collectionID: number,
-        collectionKey: Uint8Array,
-    ): Promise<EnteFile[]> {
+    // Enumerate a collection's files changed since `sinceTime`, paginating the
+    // diff from that cursor. Live rows are decrypted; tombstoned ones are
+    // surfaced as bare ids. Returns the final cursor to resume from.
+    async filesSince(args: {
+        collectionID: number;
+        collectionKey: Uint8Array;
+        sinceTime: number;
+    }): Promise<FilesPage> {
         this.assertLoggedIn();
-        const allFiles: EnteFile[] = [];
-        let sinceTime = 0;
+        const { collectionID, collectionKey } = args;
+        const files: EnteFile[] = [];
+        const deleted: number[] = [];
+        let cursor = args.sinceTime;
         for (;;) {
             const { diff, hasMore } = await this.api.getJSON<{
                 diff: RawEnteFile[];
                 hasMore: boolean;
-            }>("/collections/v2/diff", { collectionID, sinceTime });
+            }>("/collections/v2/diff", { collectionID, sinceTime: cursor });
 
+            let pageMax = cursor;
             for (const raw of diff) {
-                if (!raw.isDeleted) {
-                    allFiles.push(decryptFile(raw, collectionKey));
-                }
-                if (raw.updationTime > sinceTime) {
-                    sinceTime = raw.updationTime;
+                if (raw.isDeleted) {
+                    deleted.push(raw.id);
+                } else {
+                    files.push(decryptFile(raw, collectionKey));
                 }
+                if (raw.updationTime > pageMax) pageMax = raw.updationTime;
             }
-            if (!hasMore) break;
+
+            if (!hasMore) {
+                cursor = pageMax;
+                break;
+            }
+            // The server says there is more, but this page did not advance the
+            // cursor: following hasMore would refetch the same page forever
+            // (#7). Stop with a clear error instead of looping.
+            if (pageMax <= cursor) {
+                throw new Error(
+                    `/collections/v2/diff for collection ${collectionID} ` +
+                        `returned hasMore with a cursor that did not advance ` +
+                        `(stuck at ${cursor}); refusing to loop`,
+                );
+            }
+            cursor = pageMax;
         }
-        return allFiles;
+        return { files, deleted, cursor };
+    }
+
+    // Whole-account listing: every live collection, deletions hidden. A thin
+    // wrapper over `collectionsSince` from the beginning of time.
+    async listCollections(): Promise<Collection[]> {
+        const { collections } = await this.collectionsSince({ sinceTime: 0 });
+        return collections;
+    }
+
+    // Every live file in a collection, deletions hidden. A thin wrapper over
+    // `filesSince` from the beginning of time.
+    async listFiles(
+        collectionID: number,
+        collectionKey: Uint8Array,
+    ): Promise<EnteFile[]> {
+        const { files } = await this.filesSince({
+            collectionID,
+            collectionKey,
+            sinceTime: 0,
+        });
+        return files;
     }
 
     async downloadFile(

src/index.ts

@@ -1,6 +1,12 @@
 export const VERSION = "0.0.0";
 
-export { Client, type LoginOptions, type ClientSnapshot } from "./client.js";
+export {
+    Client,
+    type LoginOptions,
+    type ClientSnapshot,
+    type CollectionsPage,
+    type FilesPage,
+} from "./client.js";
 export {
     ApiClient,
     ApiError,