Resumable, deletion-aware collection and file enumeration (closes #38)

Adds collectionsSince/filesSince taking a starting cursor, returning the resumable max-updationTime cursor and a separate list of tombstoned ids; filesSince throws instead of looping when the server reports hasMore without advancing (closes #7). listCollections/listFiles stay as thin, unaffected wrappers.

Model: opus-4-8

src/client.ts

@@ -37,6 +37,22 @@ export interface ClientSnapshot {
     publicKey: string;
 }
 
+// The result of a resumable enumeration. Live decrypted records and deleted
+// ids are kept apart on purpose: a tombstone carries no key or metadata to
+// decrypt, so it is a bare id rather than a hollowed-out record. `cursor` is
+// the max `updationTime` seen, to pass back into the next call.
+export interface CollectionsPage {
+    collections: Collection[];
+    deleted: number[];
+    cursor: number;
+}
+
+export interface FilesPage {
+    files: EnteFile[];
+    deleted: number[];
+    cursor: number;
+}
+
 export class Client {
     private readonly api: ApiClient;
     private readonly email: string;
@@ -150,52 +166,111 @@ export class Client {
         this.api.clearAuthToken();
     }
 
-    async listCollections(): Promise<Collection[]> {
+    // Enumerate collections changed since `sinceTime`. Live collections are
+    // decrypted; tombstoned ones (isDeleted) are surfaced as bare ids. The
+    // returned cursor is the max `updationTime` seen — including tombstones, so
+    // the next sync resumes past them — falling back to `sinceTime` when the
+    // response is empty. `/collections/v2` returns the whole changed set in one
+    // response, so there is no pagination here.
+    async collectionsSince(args: {
+        sinceTime: number;
+    }): Promise<CollectionsPage> {
         this.assertLoggedIn();
-        const { collections } = await this.api.getJSON<{
+        const { collections: raws } = await this.api.getJSON<{
             collections: RawCollection[];
-        }>("/collections/v2", { sinceTime: 0 });
-        // The sync API keeps returning deleted collections as tombstones
-        // (isDeleted: true); their diff endpoint 404s, so drop them.
-        return collections
-            .filter((raw) => !raw.isDeleted)
-            .map((raw) =>
-                decryptCollection(
-                    raw,
-                    {
-                        masterKey: this.masterKey,
-                        publicKey: this.publicKey,
-                        secretKey: this.secretKey,
-                    },
-                    this.userID,
-                ),
-            );
+        }>("/collections/v2", { sinceTime: args.sinceTime });
+
+        const collections: Collection[] = [];
+        const deleted: number[] = [];
+        let cursor = args.sinceTime;
+        for (const raw of raws) {
+            if (raw.isDeleted) {
+                deleted.push(raw.id);
+            } else {
+                collections.push(
+                    decryptCollection(
+                        raw,
+                        {
+                            masterKey: this.masterKey,
+                            publicKey: this.publicKey,
+                            secretKey: this.secretKey,
+                        },
+                        this.userID,
+                    ),
+                );
+            }
+            if (raw.updationTime > cursor) cursor = raw.updationTime;
+        }
+        return { collections, deleted, cursor };
     }
 
-    async listFiles(
-        collectionID: number,
-        collectionKey: Uint8Array,
-    ): Promise<EnteFile[]> {
+    // Enumerate a collection's files changed since `sinceTime`, paginating the
+    // diff from that cursor. Live rows are decrypted; tombstoned ones are
+    // surfaced as bare ids. Returns the final cursor to resume from.
+    async filesSince(args: {
+        collectionID: number;
+        collectionKey: Uint8Array;
+        sinceTime: number;
+    }): Promise<FilesPage> {
         this.assertLoggedIn();
-        const allFiles: EnteFile[] = [];
-        let sinceTime = 0;
+        const { collectionID, collectionKey } = args;
+        const files: EnteFile[] = [];
+        const deleted: number[] = [];
+        let cursor = args.sinceTime;
         for (;;) {
             const { diff, hasMore } = await this.api.getJSON<{
                 diff: RawEnteFile[];
                 hasMore: boolean;
-            }>("/collections/v2/diff", { collectionID, sinceTime });
+            }>("/collections/v2/diff", { collectionID, sinceTime: cursor });
 
+            let pageMax = cursor;
             for (const raw of diff) {
-                if (!raw.isDeleted) {
-                    allFiles.push(decryptFile(raw, collectionKey));
-                }
-                if (raw.updationTime > sinceTime) {
-                    sinceTime = raw.updationTime;
+                if (raw.isDeleted) {
+                    deleted.push(raw.id);
+                } else {
+                    files.push(decryptFile(raw, collectionKey));
                 }
+                if (raw.updationTime > pageMax) pageMax = raw.updationTime;
             }
-            if (!hasMore) break;
+
+            if (!hasMore) {
+                cursor = pageMax;
+                break;
+            }
+            // The server says there is more, but this page did not advance the
+            // cursor: following hasMore would refetch the same page forever
+            // (#7). Stop with a clear error instead of looping.
+            if (pageMax <= cursor) {
+                throw new Error(
+                    `/collections/v2/diff for collection ${collectionID} ` +
+                        `returned hasMore with a cursor that did not advance ` +
+                        `(stuck at ${cursor}); refusing to loop`,
+                );
+            }
+            cursor = pageMax;
         }
-        return allFiles;
+        return { files, deleted, cursor };
+    }
+
+    // Whole-account listing: every live collection, deletions hidden. A thin
+    // wrapper over `collectionsSince` from the beginning of time.
+    async listCollections(): Promise<Collection[]> {
+        const { collections } = await this.collectionsSince({ sinceTime: 0 });
+        return collections;
+    }
+
+    // Every live file in a collection, deletions hidden. A thin wrapper over
+    // `filesSince` from the beginning of time.
+    async listFiles(
+        collectionID: number,
+        collectionKey: Uint8Array,
+    ): Promise<EnteFile[]> {
+        const { files } = await this.filesSince({
+            collectionID,
+            collectionKey,
+            sinceTime: 0,
+        });
+        return files;
     }
 
     async downloadFile(

src/index.ts

@@ -1,6 +1,12 @@
 export const VERSION = "0.0.0";
 
-export { Client, type LoginOptions, type ClientSnapshot } from "./client.js";
+export {
+    Client,
+    type LoginOptions,
+    type ClientSnapshot,
+    type CollectionsPage,
+    type FilesPage,
+} from "./client.js";
 export {
     ApiClient,
     ApiError,