feat(library/store): on-disk JSON metadata store for the local cache

Add `MetadataStore` (src/library/store.ts): one metadata.json holding the user id, schema version, collections cursor, and decrypted collection and file records. Loaded whole into RAM with Maps for id lookups; rewritten whole through the download layer's fsync atomic writer (temp, fsync, rename, dir fsync). Binary keys are base64-encoded on disk. A missing, unparseable, or wrong-schema file loads as an empty store, because the file is only a cache the refresh unit repopulates. Directory 0700, file 0600, matching session.json. No lock file; no sync() beyond the writer. This unit only stores; issue 42's refresh unit populates it. (closes #41) Model: opus-4-8
test(library/store): metadata store round-trip, empty-on-corrupt, perms
2026-09-22 10:20:13 +00:00 · 2026-09-22 10:18:46 +00:00
4 changed files with 47 additions and 497 deletions
@@ -18,15 +18,13 @@ Update the README API reference section to match the current implementation.
 # Completed Steps
- 2026-09-22: Added resumable, deletion-aware enumeration to `Client` (issue 38,
+- 2026-09-22: Added the on-disk JSON metadata store (`src/library/store.ts`,
-  closes issue 7). `collectionsSince`/`filesSince` take a starting cursor,
+  issue 41), phase 2 of the cache/API work. One `metadata.json` holds the user
-  decrypt live records, surface tombstoned ids in a separate `deleted` list (a
+  id, schema version, collections cursor, and the decrypted collection and file
-  tombstone has nothing to decrypt, so it is a bare id, not a hollow record),
+  records, loaded whole into RAM with `Map`s for id lookups and rewritten whole
-  and return the max `updationTime` seen as the cursor to resume from.
+  through the download layer's fsync atomic writer. Missing, corrupt, or
-  `filesSince` refuses to loop when the diff reports `hasMore` without advancing
+  wrong-schema files load empty; the directory is 0700 and the file 0600. The
-  the cursor (issue 7). `listCollections`/`listFiles` are now thin wrappers that
+  store only stores — the refresh unit (issue 42) populates it.
  enumerate from `sinceTime: 0` and drop deletions, so existing callers are
  unaffected.
 - 2026-09-22: Carried file size, thumbnail size, and the deletion flag through
  `decryptFile` (issue 37, foundation for the cache/API design). Live files now
  populate `file.size`/`thumbnail.size` from the server's `info` (left
@@ -37,22 +37,6 @@ export interface ClientSnapshot {
    publicKey: string;
 }
 // The result of a resumable enumeration. Live decrypted records and deleted
 // ids are kept apart on purpose: a tombstone carries no key or metadata to
 // decrypt, so it is a bare id rather than a hollowed-out record. `cursor` is
 // the max `updationTime` seen, to pass back into the next call.
 export interface CollectionsPage {
    collections: Collection[];
    deleted: number[];
    cursor: number;
 }
 export interface FilesPage {
    files: EnteFile[];
    deleted: number[];
    cursor: number;
 }
 export class Client {
    private readonly api: ApiClient;
    private readonly email: string;
@@ -166,111 +150,52 @@ export class Client {
        this.api.clearAuthToken();
    }
    // Enumerate collections changed since `sinceTime`. Live collections are
    // decrypted; tombstoned ones (isDeleted) are surfaced as bare ids. The
    // returned cursor is the max `updationTime` seen — including tombstones, so
    // the next sync resumes past them — falling back to `sinceTime` when the
    // response is empty. `/collections/v2` returns the whole changed set in one
    // response, so there is no pagination here.
    async collectionsSince(args: {
        sinceTime: number;
    }): Promise<CollectionsPage> {
        this.assertLoggedIn();
        const { collections: raws } = await this.api.getJSON<{
            collections: RawCollection[];
        }>("/collections/v2", { sinceTime: args.sinceTime });
        const collections: Collection[] = [];
        const deleted: number[] = [];
        let cursor = args.sinceTime;
        for (const raw of raws) {
            if (raw.isDeleted) {
                deleted.push(raw.id);
            } else {
                collections.push(
                    decryptCollection(
                        raw,
                        {
                            masterKey: this.masterKey,
                            publicKey: this.publicKey,
                            secretKey: this.secretKey,
                        },
                        this.userID,
                    ),
                );
            }
            if (raw.updationTime > cursor) cursor = raw.updationTime;
        }
        return { collections, deleted, cursor };
    }
    // Enumerate a collection's files changed since `sinceTime`, paginating the
    // diff from that cursor. Live rows are decrypted; tombstoned ones are
    // surfaced as bare ids. Returns the final cursor to resume from.
    async filesSince(args: {
        collectionID: number;
        collectionKey: Uint8Array;
        sinceTime: number;
    }): Promise<FilesPage> {
        this.assertLoggedIn();
        const { collectionID, collectionKey } = args;
        const files: EnteFile[] = [];
        const deleted: number[] = [];
        let cursor = args.sinceTime;
        for (;;) {
            const { diff, hasMore } = await this.api.getJSON<{
                diff: RawEnteFile[];
                hasMore: boolean;
            }>("/collections/v2/diff", { collectionID, sinceTime: cursor });
            let pageMax = cursor;
            for (const raw of diff) {
                if (raw.isDeleted) {
                    deleted.push(raw.id);
                } else {
                    files.push(decryptFile(raw, collectionKey));
                }
                if (raw.updationTime > pageMax) pageMax = raw.updationTime;
            }
            if (!hasMore) {
                cursor = pageMax;
                break;
            }
            // The server says there is more, but this page did not advance the
            // cursor: following hasMore would refetch the same page forever
            // (#7). Stop with a clear error instead of looping.
            if (pageMax <= cursor) {
                throw new Error(
                    `/collections/v2/diff for collection ${collectionID} ` +
                        `returned hasMore with a cursor that did not advance ` +
                        `(stuck at ${cursor}); refusing to loop`,
                );
            }
            cursor = pageMax;
        }
        return { files, deleted, cursor };
    }
    // Whole-account listing: every live collection, deletions hidden. A thin
    // wrapper over `collectionsSince` from the beginning of time.
    async listCollections(): Promise<Collection[]> {
-        const { collections } = await this.collectionsSince({ sinceTime: 0 });
+        this.assertLoggedIn();
-        return collections;
+        const { collections } = await this.api.getJSON<{
            collections: RawCollection[];
        }>("/collections/v2", { sinceTime: 0 });
        // The sync API keeps returning deleted collections as tombstones
        // (isDeleted: true); their diff endpoint 404s, so drop them.
        return collections
            .filter((raw) => !raw.isDeleted)
            .map((raw) =>
                decryptCollection(
                    raw,
                    {
                        masterKey: this.masterKey,
                        publicKey: this.publicKey,
                        secretKey: this.secretKey,
                    },
                    this.userID,
                ),
            );
    }
    // Every live file in a collection, deletions hidden. A thin wrapper over
    // `filesSince` from the beginning of time.
    async listFiles(
        collectionID: number,
        collectionKey: Uint8Array,
    ): Promise<EnteFile[]> {
-        const { files } = await this.filesSince({
+        this.assertLoggedIn();
-            collectionID,
+        const allFiles: EnteFile[] = [];
-            collectionKey,
+        let sinceTime = 0;
-            sinceTime: 0,
+        for (;;) {
-        });
+            const { diff, hasMore } = await this.api.getJSON<{
-        return files;
+                diff: RawEnteFile[];
                hasMore: boolean;
            }>("/collections/v2/diff", { collectionID, sinceTime });
            for (const raw of diff) {
                if (!raw.isDeleted) {
                    allFiles.push(decryptFile(raw, collectionKey));
                }
                if (raw.updationTime > sinceTime) {
                    sinceTime = raw.updationTime;
                }
            }
            if (!hasMore) break;
        }
        return allFiles;
    }
    async downloadFile(
@@ -1,12 +1,6 @@
 export const VERSION = "0.0.0";
-export {
+export { Client, type LoginOptions, type ClientSnapshot } from "./client.js";
    Client,
    type LoginOptions,
    type ClientSnapshot,
    type CollectionsPage,
    type FilesPage,
 } from "./client.js";
 export {
    ApiClient,
    ApiError,
@@ -1,367 +0,0 @@
 /**
 * Tests for the resumable, deletion-aware enumeration variants on `Client`:
 * `collectionsSince` and `filesSince`.
 *
 * The whole-account methods `listCollections` / `listFiles` always start at
 * `sinceTime: 0` and hide deletions. The cache refresh needs the opposite:
 * start from a saved cursor, learn what was deleted, and get back a cursor to
 * resume from next time. These two methods provide that.
 *
 * The return shape keeps live records and tombstones apart — `collections` /
 * `files` are decrypted live records, `deleted` is a plain list of the ids the
 * server tombstoned. A tombstone carries no decryptable key or metadata, so it
 * is a bare id rather than a hollowed-out `Collection` / `EnteFile`.
 *
 * All tests inject a fake `fetch` and drive a real `Client` (built with
 * `Client.fromJSON`) so the decryption path runs for real. Live rows are built
 * with libsodium exactly as the server would encrypt them; tombstone rows carry
 * only the fields the code reads (`id`, `updationTime`, `isDeleted`), because
 * they are never decrypted.
 */
 import sodium from "libsodium-wrappers-sumo";
 import { beforeAll, describe, expect, it } from "vitest";
 import { init, toBase64 } from "../../src/crypto/index.js";
 import { Client, type ClientSnapshot } from "../../src/client.js";
 // ---------------------------------------------------------------------------
 // Fixtures
 // ---------------------------------------------------------------------------
 const USER_ID = 42;
 interface Keys {
    masterKey: Uint8Array;
    publicKey: Uint8Array;
    secretKey: Uint8Array;
 }
 const buildKeys = (): Keys => {
    const kp = sodium.crypto_box_keypair();
    return {
        masterKey: sodium.crypto_secretbox_keygen(),
        publicKey: kp.publicKey,
        secretKey: kp.privateKey,
    };
 };
 const snapshotFor = (keys: Keys): ClientSnapshot => ({
    email: "user@example.com",
    userID: USER_ID,
    token: "test-token",
    masterKey: toBase64(keys.masterKey),
    secretKey: toBase64(keys.secretKey),
    publicKey: toBase64(keys.publicKey),
 });
 const secretboxEncrypt = (
    plaintext: Uint8Array,
    key: Uint8Array,
 ): { ciphertext: Uint8Array; nonce: Uint8Array } => {
    const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
    return {
        ciphertext: sodium.crypto_secretbox_easy(plaintext, nonce, key),
        nonce,
    };
 };
 /** An owned collection row as the server sends it, keyed under the master key. */
 const ownedCollectionRow = (
    masterKey: Uint8Array,
    opts: { id: number; name: string; updationTime: number },
 ): Record<string, unknown> => {
    const collectionKey = sodium.crypto_secretbox_keygen();
    const { ciphertext: encKey, nonce: keyNonce } = secretboxEncrypt(
        collectionKey,
        masterKey,
    );
    const { ciphertext: encName, nonce: nameNonce } = secretboxEncrypt(
        new TextEncoder().encode(opts.name),
        collectionKey,
    );
    return {
        id: opts.id,
        owner: { id: USER_ID },
        encryptedKey: toBase64(encKey),
        keyDecryptionNonce: toBase64(keyNonce),
        encryptedName: toBase64(encName),
        nameDecryptionNonce: toBase64(nameNonce),
        type: "album",
        updationTime: opts.updationTime,
    };
 };
 /** A live file row inside a collection, keyed under that collection's key. */
 const fileRow = (
    collectionKey: Uint8Array,
    opts: { id: number; title: string; updationTime: number },
 ): Record<string, unknown> => {
    const fileKey = sodium.crypto_secretbox_keygen();
    const { ciphertext: encFileKey, nonce: fileKeyNonce } = secretboxEncrypt(
        fileKey,
        collectionKey,
    );
    const metadata = {
        title: opts.title,
        fileType: 0,
        creationTime: opts.updationTime,
        modificationTime: opts.updationTime,
    };
    const push =
        sodium.crypto_secretstream_xchacha20poly1305_init_push(fileKey);
    const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push(
        push.state,
        new TextEncoder().encode(JSON.stringify(metadata)),
        null,
        sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
    );
    return {
        id: opts.id,
        collectionID: 1,
        ownerID: USER_ID,
        encryptedKey: toBase64(encFileKey),
        keyDecryptionNonce: toBase64(fileKeyNonce),
        metadata: {
            encryptedData: toBase64(encMeta),
            decryptionHeader: toBase64(push.header),
        },
        file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
        thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
        updationTime: opts.updationTime,
    };
 };
 /** A tombstone row. Never decrypted, so only these fields are ever read. */
 const tombstoneRow = (
    id: number,
    updationTime: number,
 ): Record<string, unknown> => ({
    id,
    updationTime,
    isDeleted: true,
 });
 const jsonResponse = (body: unknown): Response =>
    new Response(JSON.stringify(body), {
        status: 200,
        headers: { "content-type": "application/json" },
    });
 /**
 * A fetch that serves canned responses in order and records the `sinceTime`
 * query parameter each request carried, so tests can prove the cursor is
 * threaded from one page (and one call) to the next.
 */
 const recordingFetch = (
    ...responses: Response[]
 ): { fetch: typeof globalThis.fetch; sinceTimes: (string | null)[] } => {
    const sinceTimes: (string | null)[] = [];
    let i = 0;
    const fake = async (input: RequestInfo | URL): Promise<Response> => {
        const url =
            typeof input === "string"
                ? input
                : input instanceof URL
                  ? input.href
                  : input.url;
        sinceTimes.push(new URL(url).searchParams.get("sinceTime"));
        if (i >= responses.length) {
            throw new Error(`recordingFetch: no response for call #${i}`);
        }
        return responses[i++]!;
    };
    return { fetch: fake as typeof globalThis.fetch, sinceTimes };
 };
 // ---------------------------------------------------------------------------
 // Tests
 // ---------------------------------------------------------------------------
 describe("Client.filesSince", () => {
    beforeAll(async () => {
        await init();
        await sodium.ready;
    });
    it("pages from the given cursor, decrypts live rows, and collects tombstones", async () => {
        const keys = buildKeys();
        const collectionKey = sodium.crypto_secretbox_keygen();
        // Page 1 mixes a live file and a tombstone; the tombstone has the
        // higher updationTime, so it — not the live row — sets the cursor the
        // second page must be fetched from.
        const { fetch, sinceTimes } = recordingFetch(
            jsonResponse({
                diff: [
                    fileRow(collectionKey, {
                        id: 1001,
                        title: "first.jpg",
                        updationTime: 100,
                    }),
                    tombstoneRow(1002, 150),
                ],
                hasMore: true,
            }),
            jsonResponse({
                diff: [
                    fileRow(collectionKey, {
                        id: 1003,
                        title: "second.jpg",
                        updationTime: 200,
                    }),
                ],
                hasMore: false,
            }),
        );
        const client = Client.fromJSON(snapshotFor(keys), { fetch });
        const { files, deleted, cursor } = await client.filesSince({
            collectionID: 1,
            collectionKey,
            sinceTime: 0,
        });
        expect(files.map((f) => f.id)).toEqual([1001, 1003]);
        expect(files.map((f) => f.metadata.title)).toEqual([
            "first.jpg",
            "second.jpg",
        ]);
        expect(deleted).toEqual([1002]);
        expect(cursor).toBe(200);
        // First request started at the caller's cursor; the second resumed
        // from the max updationTime seen on the first page (the tombstone's).
        expect(sinceTimes).toEqual(["0", "150"]);
    });
    it("fetches only newer rows when the returned cursor is passed back in", async () => {
        const keys = buildKeys();
        const collectionKey = sodium.crypto_secretbox_keygen();
        const { fetch, sinceTimes } = recordingFetch(
            jsonResponse({ diff: [], hasMore: false }),
        );
        const client = Client.fromJSON(snapshotFor(keys), { fetch });
        const result = await client.filesSince({
            collectionID: 1,
            collectionKey,
            sinceTime: 200,
        });
        expect(result.files).toEqual([]);
        expect(result.deleted).toEqual([]);
        // An empty diff advances nothing: the cursor falls back to the input.
        expect(result.cursor).toBe(200);
        expect(sinceTimes).toEqual(["200"]);
    });
    it("stops and throws when the server claims more but does not advance (#7)", async () => {
        const keys = buildKeys();
        const collectionKey = sodium.crypto_secretbox_keygen();
        // hasMore is true, but the page's max updationTime (50) does not exceed
        // the cursor the request was made with (50). Following hasMore here
        // would refetch this same page forever.
        const { fetch, sinceTimes } = recordingFetch(
            jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }),
            jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }),
        );
        const client = Client.fromJSON(snapshotFor(keys), { fetch });
        await expect(
            client.filesSince({
                collectionID: 1,
                collectionKey,
                sinceTime: 50,
            }),
        ).rejects.toThrow(/not advance|non-advancing/i);
        // It gave up after the first page rather than looping.
        expect(sinceTimes).toEqual(["50"]);
    });
 });
 describe("Client.collectionsSince", () => {
    beforeAll(async () => {
        await init();
        await sodium.ready;
    });
    it("decrypts live collections, collects tombstones, and returns a cursor", async () => {
        const keys = buildKeys();
        const { fetch, sinceTimes } = recordingFetch(
            jsonResponse({
                collections: [
                    ownedCollectionRow(keys.masterKey, {
                        id: 1,
                        name: "Vacation",
                        updationTime: 100,
                    }),
                    tombstoneRow(3, 150),
                ],
            }),
        );
        const client = Client.fromJSON(snapshotFor(keys), { fetch });
        const { collections, deleted, cursor } = await client.collectionsSince({
            sinceTime: 0,
        });
        expect(collections.map((c) => c.id)).toEqual([1]);
        expect(collections[0]!.name).toBe("Vacation");
        expect(deleted).toEqual([3]);
        // The tombstone's updationTime advances the cursor too, so the next
        // sync starts after it rather than seeing it again.
        expect(cursor).toBe(150);
        expect(sinceTimes).toEqual(["0"]);
    });
    it("falls back to the input cursor on an empty response", async () => {
        const keys = buildKeys();
        const { fetch, sinceTimes } = recordingFetch(
            jsonResponse({ collections: [] }),
        );
        const client = Client.fromJSON(snapshotFor(keys), { fetch });
        const result = await client.collectionsSince({ sinceTime: 150 });
        expect(result.collections).toEqual([]);
        expect(result.deleted).toEqual([]);
        expect(result.cursor).toBe(150);
        expect(sinceTimes).toEqual(["150"]);
    });
 });
 describe("Client list wrappers still hide deletions", () => {
    beforeAll(async () => {
        await init();
        await sodium.ready;
    });
    it("listFiles drops tombstones and returns only live files", async () => {
        const keys = buildKeys();
        const collectionKey = sodium.crypto_secretbox_keygen();
        const { fetch, sinceTimes } = recordingFetch(
            jsonResponse({
                diff: [
                    fileRow(collectionKey, {
                        id: 7,
                        title: "keep.jpg",
                        updationTime: 100,
                    }),
                    tombstoneRow(8, 150),
                ],
                hasMore: false,
            }),
        );
        const client = Client.fromJSON(snapshotFor(keys), { fetch });
        const files = await client.listFiles(1, collectionKey);
        expect(files.map((f) => f.id)).toEqual([7]);
        // The wrapper starts a full enumeration from zero.
        expect(sinceTimes).toEqual(["0"]);
    });
 });