Plain-record library snapshot and change subscription (closes #43)

Adds a synchronous, key-free snapshot() returning LibrarySnapshot (one PhotoRecord per fileID, deduped, newest first) with edited-name/time precedence (pubMagicMetadata over basic metadata) in milliseconds, plus AlbumRecord (favorites identified by type). subscribe({onChange}) delivers LibraryChange (changed/removed albums and files, refreshedAt) only when a refresh changes something; unsubscribe stops delivery. Built on the existing refresh loop; served from RAM, safe to send over IPC.

Model: opus-4-8

test/library/records.test.ts

@@ -0,0 +1,279 @@
+/**
+ * Tests for the plain-record mapping in `src/library/records.ts` (issue #43).
+ *
+ * The library keeps decrypted `Collection`/`EnteFile` objects in RAM, but those
+ * carry binary keys and cannot cross the Electron IPC boundary. `deriveRecords`
+ * projects them into plain `AlbumRecord`/`PhotoRecord` values — no keys, no
+ * `Uint8Array`, JSON-safe — that the GUI process consumes. This file pins:
+ *
+ *  1. Field mapping from `metadata` and the two magic-metadata layers, using the
+ *     real Ente field names confirmed against the fixtures in
+ *     `test/cli/metadata-backup.test.ts` (`w`/`h`) and `test/library/store.test.ts`
+ *     (`visibility`): title/takenAt precedence, caption, width/height, geo,
+ *     visibility → isArchived/isHidden, fileType.
+ *  2. `takenAt` is milliseconds; Ente stores creationTime/editedTime in
+ *     microseconds, so the record divides by 1000.
+ *  3. Deduplication: one `PhotoRecord` per fileID even when the file belongs to
+ *     several collections, with every membership's collection id in `albumIDs`.
+ *  4. Ordering: photos and album `fileIDs` are newest first.
+ *  5. No key material survives the projection.
+ *  6. `diffRecords` reports exactly what changed between two derivations, and
+ *     returns undefined when nothing changed.
+ */
+
+import { describe, it, expect } from "vitest";
+import {
+    deriveRecords,
+    snapshotFrom,
+    diffRecords,
+} from "../../src/library/records.js";
+import type { Collection, EnteFile } from "../../src/model/types.js";
+
+const OWNER = 42;
+
+// Microsecond epoch values, as Ente stores times. 1e15 ≈ 2001 in microseconds.
+const T = (micros: number): number => micros;
+
+const collection = (
+    id: number,
+    opts: Partial<Collection> = {},
+): Collection => ({
+    id,
+    ownerID: OWNER,
+    key: new Uint8Array([id & 0xff, 1, 2, 3]),
+    name: `album-${id}`,
+    type: "album",
+    updationTime: T(1_700_000_000_000_000),
+    isShared: false,
+    ...opts,
+});
+
+const file = (
+    id: number,
+    collectionID: number,
+    opts: Partial<EnteFile> & {
+        creationTime?: number;
+        title?: string;
+    } = {},
+): EnteFile => {
+    const { creationTime, title, ...rest } = opts;
+    return {
+        id,
+        collectionID,
+        ownerID: OWNER,
+        key: new Uint8Array([id & 0xff, 9, 8, 7]),
+        metadata: {
+            title: title ?? `file-${id}.jpg`,
+            fileType: "image",
+            creationTime: creationTime ?? T(1_700_000_000_000_000),
+            modificationTime: T(1_700_000_000_000_000),
+        },
+        file: { decryptionHeader: "aGVhZGVy" },
+        thumbnail: { decryptionHeader: "dGh1bWI=" },
+        updationTime: T(1_700_000_000_000_000),
+        ...rest,
+    };
+};
+
+describe("deriveRecords: photo mapping", () => {
+    it("projects a file into a plain PhotoRecord with no key material", () => {
+        const f = file(1001, 1, {
+            metadata: {
+                title: "IMG_1.jpg",
+                fileType: "image",
+                creationTime: T(1_699_000_000_000_000),
+                modificationTime: T(1_699_000_000_000_000),
+                latitude: 52.52,
+                longitude: 13.405,
+            },
+        });
+        const { photos } = deriveRecords([collection(1)], [f]);
+        const rec = photos.get(1001)!;
+
+        expect(rec.fileID).toBe(1001);
+        expect(rec.albumIDs).toEqual([1]);
+        expect(rec.title).toBe("IMG_1.jpg");
+        expect(rec.fileType).toBe("image");
+        expect(rec.latitude).toBeCloseTo(52.52);
+        expect(rec.longitude).toBeCloseTo(13.405);
+        expect(rec.isArchived).toBe(false);
+        expect(rec.isHidden).toBe(false);
+
+        // Safe to send over IPC: no key, no Uint8Array, JSON round-trips whole.
+        expect("key" in rec).toBe(false);
+        expect(JSON.parse(JSON.stringify(rec))).toEqual(rec);
+    });
+
+    it("takenAt is creationTime converted from microseconds to milliseconds", () => {
+        const f = file(1001, 1, { creationTime: T(1_699_000_000_000_000) });
+        const { photos } = deriveRecords([collection(1)], [f]);
+        expect(photos.get(1001)!.takenAt).toBe(1_699_000_000_000);
+    });
+
+    it("prefers pubMagicMetadata.editedName and editedTime over metadata", () => {
+        const f = file(1001, 1, {
+            title: "original.jpg",
+            creationTime: T(1_699_000_000_000_000),
+            pubMagicMetadata: {
+                editedName: "Sunset over the bay",
+                editedTime: T(1_650_000_000_000_000),
+            },
+        });
+        const { photos } = deriveRecords([collection(1)], [f]);
+        const rec = photos.get(1001)!;
+        expect(rec.title).toBe("Sunset over the bay");
+        expect(rec.takenAt).toBe(1_650_000_000_000);
+    });
+
+    it("falls back to metadata when the edited fields are empty or absent", () => {
+        const f = file(1001, 1, {
+            title: "original.jpg",
+            creationTime: T(1_699_000_000_000_000),
+            pubMagicMetadata: { editedName: "" },
+        });
+        const { photos } = deriveRecords([collection(1)], [f]);
+        const rec = photos.get(1001)!;
+        expect(rec.title).toBe("original.jpg");
+        expect(rec.takenAt).toBe(1_699_000_000_000);
+    });
+
+    it("maps caption and width/height from the public magic metadata", () => {
+        const f = file(1001, 1, {
+            pubMagicMetadata: {
+                caption: "at the beach",
+                w: 3000,
+                h: 2000,
+            },
+        });
+        const rec = deriveRecords([collection(1)], [f]).photos.get(1001)!;
+        expect(rec.caption).toBe("at the beach");
+        expect(rec.width).toBe(3000);
+        expect(rec.height).toBe(2000);
+    });
+
+    it("omits optional fields that are absent from the metadata", () => {
+        const rec = deriveRecords([collection(1)], [file(1001, 1)]).photos.get(
+            1001,
+        )!;
+        expect("caption" in rec).toBe(false);
+        expect("width" in rec).toBe(false);
+        expect("height" in rec).toBe(false);
+        expect("latitude" in rec).toBe(false);
+    });
+
+    it("reads archived and hidden from private magicMetadata.visibility", () => {
+        const archived = file(1, 1, { magicMetadata: { visibility: 1 } });
+        const hidden = file(2, 1, { magicMetadata: { visibility: 2 } });
+        const visible = file(3, 1, { magicMetadata: { visibility: 0 } });
+        const { photos } = deriveRecords(
+            [collection(1)],
+            [archived, hidden, visible],
+        );
+        expect(photos.get(1)).toMatchObject({
+            isArchived: true,
+            isHidden: false,
+        });
+        expect(photos.get(2)).toMatchObject({
+            isArchived: false,
+            isHidden: true,
+        });
+        expect(photos.get(3)).toMatchObject({
+            isArchived: false,
+            isHidden: false,
+        });
+    });
+});
+
+describe("deriveRecords: dedup and ordering", () => {
+    it("emits one PhotoRecord per fileID across memberships, all albums listed", () => {
+        // File 1001 belongs to collections 1 and 2; 2002 only to 2.
+        const files = [
+            file(1001, 1, { creationTime: T(1_700_000_000_000_000) }),
+            file(1001, 2, { creationTime: T(1_700_000_000_000_000) }),
+            file(2002, 2, { creationTime: T(1_710_000_000_000_000) }),
+        ];
+        const { photos } = deriveRecords([collection(1), collection(2)], files);
+        expect([...photos.keys()].sort((a, b) => a - b)).toEqual([1001, 2002]);
+        expect(photos.get(1001)!.albumIDs).toEqual([1, 2]);
+        expect(photos.get(2002)!.albumIDs).toEqual([2]);
+    });
+
+    it("orders snapshot photos newest first by takenAt", () => {
+        const files = [
+            file(1, 1, { creationTime: T(1_600_000_000_000_000) }),
+            file(2, 1, { creationTime: T(1_800_000_000_000_000) }),
+            file(3, 1, { creationTime: T(1_700_000_000_000_000) }),
+        ];
+        const snap = snapshotFrom(deriveRecords([collection(1)], files), 123);
+        expect(snap.photos.map((p) => p.fileID)).toEqual([2, 3, 1]);
+        expect(snap.takenAt).toBe(123);
+    });
+
+    it("orders album fileIDs newest first", () => {
+        const files = [
+            file(1, 7, { creationTime: T(1_600_000_000_000_000) }),
+            file(2, 7, { creationTime: T(1_800_000_000_000_000) }),
+            file(3, 7, { creationTime: T(1_700_000_000_000_000) }),
+        ];
+        const { albums } = deriveRecords([collection(7)], files);
+        expect(albums.get(7)!.fileIDs).toEqual([2, 3, 1]);
+    });
+});
+
+describe("deriveRecords: album mapping", () => {
+    it("carries collection identity, sharing, and the favorites type", () => {
+        const fav = collection(9, {
+            name: "Favorites",
+            type: "favorites",
+            isShared: true,
+            updationTime: T(1_705_000_000_000_000),
+        });
+        const rec = deriveRecords([fav], [file(1, 9)]).albums.get(9)!;
+        expect(rec).toMatchObject({
+            collectionID: 9,
+            name: "Favorites",
+            type: "favorites",
+            isShared: true,
+            updationTime: T(1_705_000_000_000_000),
+        });
+        expect("key" in rec).toBe(false);
+        expect(JSON.parse(JSON.stringify(rec))).toEqual(rec);
+    });
+});
+
+describe("diffRecords", () => {
+    const at = 999;
+
+    it("returns undefined when nothing changed", () => {
+        const a = deriveRecords([collection(1)], [file(1, 1)]);
+        const b = deriveRecords([collection(1)], [file(1, 1)]);
+        expect(diffRecords(a, b, at)).toBeUndefined();
+    });
+
+    it("reports added and changed albums and photos and removals", () => {
+        const before = deriveRecords(
+            [collection(1), collection(2)],
+            [file(1, 1), file(2, 2)],
+        );
+        // Collection 2 is gone (album + its only file removed). Collection 1 is
+        // renamed (changed album), gains file 3, and file 1 is retitled.
+        const after = deriveRecords(
+            [collection(1, { name: "renamed" })],
+            [
+                file(1, 1, {
+                    pubMagicMetadata: { editedName: "new title" },
+                }),
+                file(3, 1),
+            ],
+        );
+        const change = diffRecords(before, after, at)!;
+        expect(change.refreshedAt).toBe(at);
+        expect(change.albumIDsRemoved).toEqual([2]);
+        expect(change.fileIDsRemoved).toEqual([2]);
+        expect(change.albumsChanged.map((a) => a.collectionID)).toEqual([1]);
+        expect(change.albumsChanged[0]!.name).toBe("renamed");
+        expect(
+            change.photosChanged.map((p) => p.fileID).sort((x, y) => x - y),
+        ).toEqual([1, 3]);
+    });
+});

test/library/snapshot.test.ts

@@ -0,0 +1,303 @@
+/**
+ * Tests for `Library.snapshot()` and `Library.subscribe()` (issue #43).
+ *
+ * These are the surface the GUI consumes across Electron IPC. `snapshot()` is
+ * synchronous — it reads the in-RAM store and projects it into plain records
+ * (no keys) — and `subscribe({ onChange })` delivers a `LibraryChange` whenever
+ * a background refresh actually changes the derived records. The contracts:
+ *
+ *  1. `snapshot()` deduplicates a file across memberships into one record with
+ *     every album id, orders photos newest first, and carries no key material.
+ *  2. `subscribe` fires on a refresh that changes something, with the exact
+ *     changed and removed sets for both albums and photos.
+ *  3. A refresh that changes nothing (an empty diff) fires no change.
+ *  4. `unsubscribe()` stops further delivery.
+ *
+ * The client is the same scripted mock used by the refresh-loop tests: no
+ * crypto, no network. Interval tests use a short real interval and `vi.waitFor`.
+ */
+
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { Library } from "../../src/library/index.js";
+import type { LibraryChange } from "../../src/library/records.js";
+import type { CollectionsPage, FilesPage } from "../../src/client.js";
+import type { Collection, EnteFile } from "../../src/model/types.js";
+
+const USER_ID = 42;
+const FAST_INTERVAL = 0.02;
+
+const collection = (
+    id: number,
+    updationTime: number,
+    name = `album-${id}`,
+): Collection => ({
+    id,
+    ownerID: USER_ID,
+    key: new Uint8Array([id & 0xff]),
+    name,
+    type: "album",
+    updationTime,
+    isShared: false,
+});
+
+const file = (
+    id: number,
+    collectionID: number,
+    creationTime: number,
+): EnteFile => ({
+    id,
+    collectionID,
+    ownerID: USER_ID,
+    key: new Uint8Array([id & 0xff]),
+    metadata: {
+        title: `file-${id}.jpg`,
+        fileType: "image",
+        creationTime,
+        modificationTime: creationTime,
+    },
+    file: { decryptionHeader: "aGVhZGVy" },
+    thumbnail: { decryptionHeader: "dGh1bWI=" },
+    updationTime: creationTime,
+});
+
+class MockClient {
+    userID = USER_ID;
+    collectionsQueue: CollectionsPage[] = [];
+    filesByCollection = new Map<number, FilesPage[]>();
+
+    whoami(): { email: string; userID: number } {
+        return { email: "user@example.com", userID: this.userID };
+    }
+
+    async collectionsSince(args: {
+        sinceTime: number;
+    }): Promise<CollectionsPage> {
+        return (
+            this.collectionsQueue.shift() ?? {
+                collections: [],
+                deleted: [],
+                cursor: args.sinceTime,
+            }
+        );
+    }
+
+    async filesSince(args: {
+        collectionID: number;
+        collectionKey: Uint8Array;
+        sinceTime: number;
+    }): Promise<FilesPage> {
+        const queue = this.filesByCollection.get(args.collectionID);
+        return (
+            queue?.shift() ?? {
+                files: [],
+                deleted: [],
+                cursor: args.sinceTime,
+            }
+        );
+    }
+
+    filesFor(collectionID: number, ...pages: FilesPage[]): void {
+        this.filesByCollection.set(collectionID, pages);
+    }
+}
+
+describe("Library.snapshot and Library.subscribe", () => {
+    let dir: string;
+    let cacheDirectory: string;
+
+    beforeEach(() => {
+        dir = mkdtempSync(join(tmpdir(), "quak-snapshot-"));
+        cacheDirectory = join(dir, "cache");
+    });
+
+    afterEach(() => {
+        rmSync(dir, { recursive: true, force: true });
+    });
+
+    it("snapshot() dedupes across memberships, orders newest first, holds no keys", async () => {
+        const client = new MockClient();
+        client.collectionsQueue.push({
+            collections: [collection(1, 100), collection(2, 100)],
+            deleted: [],
+            cursor: 100,
+        });
+        // File 1001 is in both collections; 2002 only in collection 2 and newer.
+        client.filesFor(1, {
+            files: [file(1001, 1, 1_600_000_000_000_000)],
+            deleted: [],
+            cursor: 1_600_000_000_000_000,
+        });
+        client.filesFor(2, {
+            files: [
+                file(1001, 2, 1_600_000_000_000_000),
+                file(2002, 2, 1_800_000_000_000_000),
+            ],
+            deleted: [],
+            cursor: 1_800_000_000_000_000,
+        });
+
+        const lib = await Library.open({ client, cacheDirectory });
+        try {
+            const snap = lib.snapshot();
+
+            // One record per fileID, newest first, both albums on the shared file.
+            expect(snap.photos.map((p) => p.fileID)).toEqual([2002, 1001]);
+            const shared = snap.photos.find((p) => p.fileID === 1001)!;
+            expect(shared.albumIDs).toEqual([1, 2]);
+            expect(shared.takenAt).toBe(1_600_000_000_000);
+
+            expect(snap.albums.map((a) => a.collectionID).sort()).toEqual([
+                1, 2,
+            ]);
+
+            // Nothing carries key material; the whole snapshot is JSON-safe.
+            expect(JSON.parse(JSON.stringify(snap))).toEqual(snap);
+            for (const p of snap.photos) expect("key" in p).toBe(false);
+            for (const a of snap.albums) expect("key" in a).toBe(false);
+        } finally {
+            lib.close();
+        }
+    });
+
+    it("subscribe fires on a refresh change with the correct changed/removed sets", async () => {
+        const client = new MockClient();
+        client.collectionsQueue.push({
+            collections: [collection(1, 100), collection(2, 100)],
+            deleted: [],
+            cursor: 100,
+        });
+        client.filesFor(1, {
+            files: [file(1001, 1, 1_600_000_000_000_000)],
+            deleted: [],
+            cursor: 1_600_000_000_000_000,
+        });
+        client.filesFor(2, {
+            files: [file(2002, 2, 1_600_000_000_000_000)],
+            deleted: [],
+            cursor: 1_600_000_000_000_000,
+        });
+
+        const lib = await Library.open({
+            client,
+            cacheDirectory,
+            refreshIntervalSeconds: FAST_INTERVAL,
+        });
+        const changes: LibraryChange[] = [];
+        const { unsubscribe } = lib.subscribe({
+            onChange: (c) => changes.push(c),
+        });
+        try {
+            // Next refresh: collection 2 (and its file) tombstoned; collection 1
+            // gains file 1003.
+            client.filesFor(1, {
+                files: [file(1003, 1, 1_700_000_000_000_000)],
+                deleted: [],
+                cursor: 1_700_000_000_000_000,
+            });
+            client.collectionsQueue.push({
+                collections: [collection(1, 200)],
+                deleted: [2],
+                cursor: 200,
+            });
+
+            await vi.waitFor(() => expect(changes.length).toBeGreaterThan(0), {
+                timeout: 2000,
+                interval: 5,
+            });
+
+            const change = changes[0]!;
+            expect(change.albumIDsRemoved).toEqual([2]);
+            expect(change.fileIDsRemoved).toEqual([2002]);
+            expect(change.photosChanged.map((p) => p.fileID)).toEqual([1003]);
+            expect(change.albumsChanged.map((a) => a.collectionID)).toEqual([
+                1,
+            ]);
+            expect(change.refreshedAt).toBeGreaterThan(0);
+        } finally {
+            unsubscribe();
+            lib.close();
+        }
+    });
+
+    it("a refresh that changes nothing fires no change", async () => {
+        const client = new MockClient();
+        client.collectionsQueue.push({
+            collections: [collection(1, 100)],
+            deleted: [],
+            cursor: 100,
+        });
+        client.filesFor(1, {
+            files: [file(1001, 1, 1_600_000_000_000_000)],
+            deleted: [],
+            cursor: 1_600_000_000_000_000,
+        });
+
+        const lib = await Library.open({
+            client,
+            cacheDirectory,
+            refreshIntervalSeconds: FAST_INTERVAL,
+        });
+        const changes: LibraryChange[] = [];
+        const { unsubscribe } = lib.subscribe({
+            onChange: (c) => changes.push(c),
+        });
+        try {
+            // Let several empty-diff ticks pass; none may deliver a change.
+            await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 6));
+            expect(changes).toEqual([]);
+        } finally {
+            unsubscribe();
+            lib.close();
+        }
+    });
+
+    it("unsubscribe stops further delivery", async () => {
+        const client = new MockClient();
+        client.collectionsQueue.push({
+            collections: [collection(1, 100)],
+            deleted: [],
+            cursor: 100,
+        });
+        client.filesFor(1, {
+            files: [file(1001, 1, 1_600_000_000_000_000)],
+            deleted: [],
+            cursor: 1_600_000_000_000_000,
+        });
+
+        const lib = await Library.open({
+            client,
+            cacheDirectory,
+            refreshIntervalSeconds: FAST_INTERVAL,
+        });
+        const changes: LibraryChange[] = [];
+        const { unsubscribe } = lib.subscribe({
+            onChange: (c) => changes.push(c),
+        });
+        unsubscribe();
+        try {
+            client.collectionsQueue.push({
+                collections: [collection(3, 300)],
+                deleted: [],
+                cursor: 300,
+            });
+            client.filesFor(3, {
+                files: [file(3003, 3, 1_700_000_000_000_000)],
+                deleted: [],
+                cursor: 1_700_000_000_000_000,
+            });
+            // The change lands in the store, but the cancelled subscriber sees
+            // nothing.
+            await vi.waitFor(
+                () => expect(lib.snapshot().albums.length).toBe(2),
+                { timeout: 2000, interval: 5 },
+            );
+            expect(changes).toEqual([]);
+        } finally {
+            lib.close();
+        }
+    });
+});