/** * Tests for `Library.open()` and its transparent background refresh loop. * * The library keeps the account's server state in a `MetadataStore` (issue * #41) and pulls changes with the resumable, tombstone-aware enumerators on * `Client` (issue #38: `collectionsSince` / `filesSince`). `open()` loads the * cache, does one refresh, then refreshes again every `refreshIntervalSeconds` * on a background timer. The design (#36) forbids an exposed `sync()`, a * `serverReachable` flag, a `lib.refresh()` method, and a "before each read" * mode. The contracts exercised here: * * 1. Reads are answered from RAM. A read never calls the client. * 2. `open()` does an initial refresh, then the interval keeps refreshing; * each refresh resumes from the stored cursor and applies diffs + tombstones. * 3. The cache is rewritten only when a refresh actually changes something. * 4. A failed refresh is invisible to reads: the last good data stays, the * failure surfaces via `onProgress` ("failed") and `status()`, and a later * success clears the error. `open()` itself resolves even when the first * refresh fails (offline start from cache). * 5. `close()` stops the timer and is idempotent. * 6. `cacheDirectory` defaults to the env-paths cache dir plus the user id. * 7. `open()` branches on the cache: an empty cache awaits the first refresh * (it has nothing to serve yet); an existing cache serves its copy at once * and refreshes in the background, so a slow or dead server never stalls * opening. * 8. A save failure that leaves RAM ahead of disk keeps `status().lastError` * set and keeps retrying the write; a later empty refresh does not clear it. * * The client is a mock: no crypto, no network. It serves scripted pages and * records the `sinceTime` each call carried so cursor threading is provable. * * On an empty cache `open()` awaits the initial refresh (including its cache * write), so state right after `open()` is deterministic; the tests that * inspect post-`open()` state seed no cache and rely on that. Tests for an * existing-cache open seed a store first and prove `open()` returns without * waiting for the network. The interval tests then use real timers with a * short interval and `vi.waitFor`: a fake clock cannot settle the real * fsync-and-rename cache write, and empty diffs never write, so the eventual * state is stable to poll for. * * A refresh changes RAM before it writes the cache file, so a polled state can * be visible while that write is still running. Every test therefore awaits * `close()`, which waits for the in-flight refresh, before `afterEach` removes * the directory. */ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import envPaths from "env-paths"; import { Library, type RefreshEvent } from "../../src/library/index.js"; import { MetadataStore } from "../../src/library/store.js"; import type { CollectionsPage, FilesPage } from "../../src/client.js"; import type { Collection, EnteFile } from "../../src/model/types.js"; const USER_ID = 42; // Short enough that a couple of ticks pass within a test, long enough not to // spin; interval tests poll for the eventual state rather than counting ticks. const FAST_INTERVAL = 0.02; const collection = ( id: number, updationTime: number, name = `album-${id}`, ): Collection => ({ id, ownerID: USER_ID, key: new Uint8Array([id & 0xff]), name, type: "album", updationTime, isShared: false, }); const file = ( id: number, collectionID: number, updationTime: number, ): EnteFile => ({ id, collectionID, ownerID: USER_ID, key: new Uint8Array([id & 0xff]), metadata: { title: `file-${id}.jpg`, fileType: "image", creationTime: updationTime, modificationTime: updationTime, }, file: { decryptionHeader: "aGVhZGVy" }, thumbnail: { decryptionHeader: "dGh1bWI=" }, updationTime, }); /** * A mock `Client`. `collectionsSince` shifts one page off `collectionsQueue` * per call (an empty diff that advances nothing when the queue runs dry); * `filesSince` shifts from a per-collection queue. `failCollections` makes the * next and all further collection fetches throw, to simulate an offline server. */ class MockClient { userID = USER_ID; failCollections = false; collectionsQueue: CollectionsPage[] = []; filesByCollection = new Map(); collectionsSinceTimes: number[] = []; filesCalls: { collectionID: number; sinceTime: number }[] = []; whoami(): { email: string; userID: number } { return { email: "user@example.com", userID: this.userID }; } async collectionsSince(args: { sinceTime: number; }): Promise { this.collectionsSinceTimes.push(args.sinceTime); if (this.failCollections) throw new Error("network down"); return ( this.collectionsQueue.shift() ?? { collections: [], deleted: [], cursor: args.sinceTime, } ); } async filesSince(args: { collectionID: number; collectionKey: Uint8Array; sinceTime: number; }): Promise { this.filesCalls.push({ collectionID: args.collectionID, sinceTime: args.sinceTime, }); const queue = this.filesByCollection.get(args.collectionID); return ( queue?.shift() ?? { files: [], deleted: [], cursor: args.sinceTime, } ); } filesFor(collectionID: number, ...pages: FilesPage[]): void { this.filesByCollection.set(collectionID, pages); } } describe("Library.open and background refresh", () => { let dir: string; let cacheDirectory: string; beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "quak-library-")); cacheDirectory = join(dir, "cache"); }); afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); it("does an initial refresh and answers reads from the cache", async () => { const client = new MockClient(); client.collectionsQueue.push({ collections: [collection(1, 100)], deleted: [], cursor: 100, }); client.filesFor(1, { files: [file(1001, 1, 90), file(1002, 1, 95)], deleted: [], cursor: 95, }); const lib = await Library.open({ client, cacheDirectory }); try { expect(lib.listCollections().map((c) => c.id)).toEqual([1]); expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001, 1002]); expect(lib.getFile(1, 1001)?.metadata.title).toBe("file-1001.jpg"); const status = lib.status(); expect(status.userID).toBe(USER_ID); expect(status.collections).toBe(1); expect(status.files).toBe(2); expect(status.lastRefreshAt).toBeGreaterThan(0); expect(status.lastError).toBeUndefined(); // The initial refresh persisted the cache to disk. const reloaded = await MetadataStore.load( join(cacheDirectory, "metadata.json"), ); expect(reloaded.getFile(1, 1001)?.id).toBe(1001); expect(reloaded.collectionsSinceTime).toBe(100); } finally { await lib.close(); } }); it("reads never call the client", async () => { const client = new MockClient(); client.collectionsQueue.push({ collections: [collection(1, 100)], deleted: [], cursor: 100, }); client.filesFor(1, { files: [file(1001, 1, 90)], deleted: [], cursor: 90, }); const lib = await Library.open({ client, cacheDirectory }); try { const collectionCalls = client.collectionsSinceTimes.length; const fileCalls = client.filesCalls.length; lib.listCollections(); lib.getCollection(1); lib.listFiles(1); lib.getFile(1, 1001); lib.status(); expect(client.collectionsSinceTimes.length).toBe(collectionCalls); expect(client.filesCalls.length).toBe(fileCalls); } finally { await lib.close(); } }); it("resumes each refresh from the stored cursor", async () => { // Seed a cache with a cursor and a collection, as a prior run left it. const path = join(cacheDirectory, "metadata.json"); const seed = await MetadataStore.load(path); seed.userID = USER_ID; seed.collectionsSinceTime = 500; seed.putCollection(collection(1, 400)); seed.putFile(file(1001, 1, 400)); await seed.save(); const client = new MockClient(); // The collection's updationTime advances (400 -> 600), so its files are // re-enumerated from the collection's stored updationTime (400). client.collectionsQueue.push({ collections: [collection(1, 600)], deleted: [], cursor: 600, }); client.filesFor(1, { files: [file(1002, 1, 550)], deleted: [], cursor: 550, }); // Opening from an existing cache serves the seeded copy at once and // refreshes in the background, so the refresh's effects are polled for. const lib = await Library.open({ client, cacheDirectory }); try { await vi.waitFor( () => { // Collections resumed from the stored cursor, and files were // re-enumerated from the stored collection updationTime. expect(client.collectionsSinceTimes[0]).toBe(500); expect(client.filesCalls).toEqual([ { collectionID: 1, sinceTime: 400 }, ]); expect(lib.listFiles(1).map((f) => f.id)).toEqual([ 1001, 1002, ]); }, { timeout: 2000, interval: 5 }, ); } finally { await lib.close(); } }); it("does not re-enumerate a collection whose updationTime did not advance", async () => { const path = join(cacheDirectory, "metadata.json"); const seed = await MetadataStore.load(path); seed.userID = USER_ID; seed.collectionsSinceTime = 100; seed.putCollection(collection(1, 400)); await seed.save(); const client = new MockClient(); // The collection comes back in the diff (its metadata changed) but at // the same updationTime, so its files must not be re-fetched. client.collectionsQueue.push({ collections: [collection(1, 400, "renamed")], deleted: [], cursor: 400, }); // Existing cache: the rename lands via the background refresh. const lib = await Library.open({ client, cacheDirectory }); try { await vi.waitFor( () => expect(lib.getCollection(1)?.name).toBe("renamed"), { timeout: 2000, interval: 5 }, ); // The collection's updationTime did not advance, so its files were // never re-fetched. expect(client.filesCalls).toEqual([]); } finally { await lib.close(); } }); it("applies diffs and tombstones on the interval", async () => { const client = new MockClient(); client.collectionsQueue.push({ collections: [collection(1, 100), collection(2, 100)], deleted: [], cursor: 100, }); client.filesFor(1, { files: [file(1001, 1, 90)], deleted: [], cursor: 90, }); client.filesFor(2, { files: [file(2001, 2, 90)], deleted: [], cursor: 90, }); const lib = await Library.open({ client, cacheDirectory, refreshIntervalSeconds: FAST_INTERVAL, }); try { expect(lib.listCollections().map((c) => c.id)).toEqual([1, 2]); expect(lib.listFiles(2).map((f) => f.id)).toEqual([2001]); // Next refresh: collection 2 is tombstoned; collection 1 gains a // file and loses its old one. client.filesFor(1, { files: [file(1002, 1, 190)], deleted: [1001], cursor: 190, }); client.collectionsQueue.push({ collections: [collection(1, 200)], deleted: [2], cursor: 200, }); await vi.waitFor( () => { expect(lib.listCollections().map((c) => c.id)).toEqual([1]); expect(lib.listFiles(1).map((f) => f.id)).toEqual([1002]); // Collection 2's files went with it. expect(lib.listFiles(2)).toEqual([]); }, { timeout: 2000, interval: 5 }, ); } finally { await lib.close(); } }); it("rewrites the cache only when a refresh changes something", async () => { const saveSpy = vi.spyOn(MetadataStore.prototype, "save"); const client = new MockClient(); client.collectionsQueue.push({ collections: [collection(1, 100)], deleted: [], cursor: 100, }); client.filesFor(1, { files: [file(1001, 1, 90)], deleted: [], cursor: 90, }); const lib = await Library.open({ client, cacheDirectory, refreshIntervalSeconds: FAST_INTERVAL, }); try { // The initial refresh changed everything, so it saved once. expect(saveSpy).toHaveBeenCalledTimes(1); // Several empty-diff ticks pass; none of them may rewrite the file. await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 4)); expect(saveSpy).toHaveBeenCalledTimes(1); // A real change triggers exactly one more rewrite; later empty ticks // still do not, so the count settles at two. client.collectionsQueue.push({ collections: [collection(2, 200)], deleted: [], cursor: 200, }); await vi.waitFor(() => expect(saveSpy).toHaveBeenCalledTimes(2), { timeout: 2000, interval: 5, }); await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 4)); expect(saveSpy).toHaveBeenCalledTimes(2); } finally { await lib.close(); saveSpy.mockRestore(); } }); it("keeps a failed refresh invisible to reads and recovers later", async () => { const events: RefreshEvent[] = []; const client = new MockClient(); client.collectionsQueue.push({ collections: [collection(1, 100)], deleted: [], cursor: 100, }); client.filesFor(1, { files: [file(1001, 1, 90)], deleted: [], cursor: 90, }); const lib = await Library.open({ client, cacheDirectory, refreshIntervalSeconds: FAST_INTERVAL, onProgress: (e) => events.push(e), }); try { expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]); // The server goes away; refreshes now fail. client.failCollections = true; await vi.waitFor( () => expect(lib.status().lastError).toMatch(/network down/), { timeout: 2000, interval: 5 }, ); // Reads still see the last good data; the failure was reported. expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]); expect( events.some( (e) => e.operation === "refresh" && e.status === "failed", ), ).toBe(true); // Recovery: a later refresh succeeds and clears the error. client.failCollections = false; client.collectionsQueue.push({ collections: [collection(2, 300)], deleted: [], cursor: 300, }); await vi.waitFor( () => { expect(lib.status().lastError).toBeUndefined(); expect(lib.listCollections().map((c) => c.id)).toEqual([ 1, 2, ]); }, { timeout: 2000, interval: 5 }, ); } finally { await lib.close(); } }); it("resolves open() even when the first refresh fails", async () => { const client = new MockClient(); client.failCollections = true; const events: RefreshEvent[] = []; const lib = await Library.open({ client, cacheDirectory, onProgress: (e) => events.push(e), }); try { // Nothing was cached and the server is unreachable: reads are empty, // but the library opened and the failure is on record. expect(lib.listCollections()).toEqual([]); expect(lib.status().lastError).toMatch(/network down/); expect(lib.status().lastRefreshAt).toBeUndefined(); expect( events.some( (e) => e.operation === "refresh" && e.status === "failed", ), ).toBe(true); } finally { await lib.close(); } }); it("opens from an existing cache without waiting for the first refresh", async () => { // Seed a cache as a prior run left it. const path = join(cacheDirectory, "metadata.json"); const seed = await MetadataStore.load(path); seed.userID = USER_ID; seed.collectionsSinceTime = 500; seed.putCollection(collection(1, 400)); seed.putFile(file(1001, 1, 400)); await seed.save(); // The server does not answer this run's first refresh until the test // is done with it. let answerFirstFetch: (page: CollectionsPage) => void = () => {}; const client = new MockClient(); client.collectionsSince = () => new Promise((resolve) => { answerFirstFetch = resolve; }); // open() must resolve from the cache without blocking on the network, // and reads must serve the seeded copy. const lib = await Library.open({ client, cacheDirectory }); try { expect(lib.listCollections().map((c) => c.id)).toEqual([1]); expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]); // The first refresh is still outstanding: nothing has completed or // failed yet. expect(lib.status().lastRefreshAt).toBeUndefined(); expect(lib.status().lastError).toBeUndefined(); } finally { // close() waits for the outstanding refresh, so let it finish. answerFirstFetch({ collections: [], deleted: [], cursor: 500 }); await lib.close(); } }); it("awaits the first refresh on a first run with an empty cache", async () => { // No cache on disk: open() must not resolve until the first fetch does, // so it never hands back an empty library it could have filled. let releaseFirstFetch: (page: CollectionsPage) => void = () => {}; const gate = new Promise((resolve) => { releaseFirstFetch = resolve; }); const client = new MockClient(); client.filesFor(1, { files: [file(1001, 1, 90)], deleted: [], cursor: 90, }); client.collectionsSince = async (args: { sinceTime: number }) => { client.collectionsSinceTimes.push(args.sinceTime); return gate; }; let opened = false; const openPromise = Library.open({ client, cacheDirectory }).then( (l) => { opened = true; return l; }, ); // While the first fetch is outstanding, open() has not resolved. await new Promise((r) => setTimeout(r, 20)); expect(opened).toBe(false); // Completing the fetch lets open() resolve with the data in place. releaseFirstFetch({ collections: [collection(1, 100)], deleted: [], cursor: 100, }); const lib = await openPromise; try { expect(opened).toBe(true); expect(lib.listCollections().map((c) => c.id)).toEqual([1]); expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]); expect(lib.status().lastRefreshAt).toBeGreaterThan(0); } finally { await lib.close(); } }); it("keeps a save failure visible until a save actually succeeds", async () => { const saveSpy = vi .spyOn(MetadataStore.prototype, "save") .mockRejectedValue(new Error("disk full")); const client = new MockClient(); client.collectionsQueue.push({ collections: [collection(1, 100)], deleted: [], cursor: 100, }); client.filesFor(1, { files: [file(1001, 1, 90)], deleted: [], cursor: 90, }); const lib = await Library.open({ client, cacheDirectory, refreshIntervalSeconds: FAST_INTERVAL, }); try { // The initial refresh mutated RAM but its save failed, so the error // is on record and no refresh has counted as successful. expect(lib.status().lastError).toMatch(/disk full/); expect(lib.status().lastRefreshAt).toBeUndefined(); // Empty-diff ticks pass. Each still retries the unsaved write and // still fails, so the error never silently clears and the refresh // clock never advances — RAM must not run ahead of disk unnoticed. const savesBefore = saveSpy.mock.calls.length; await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 4)); expect(saveSpy.mock.calls.length).toBeGreaterThan(savesBefore); expect(lib.status().lastError).toMatch(/disk full/); expect(lib.status().lastRefreshAt).toBeUndefined(); // Once the disk recovers, the next tick persists the pending change // and only then clears the error and advances the clock. saveSpy.mockRestore(); await vi.waitFor( () => { expect(lib.status().lastError).toBeUndefined(); expect(lib.status().lastRefreshAt).toBeGreaterThan(0); }, { timeout: 2000, interval: 5 }, ); const reloaded = await MetadataStore.load( join(cacheDirectory, "metadata.json"), ); expect(reloaded.getFile(1, 1001)?.id).toBe(1001); } finally { await lib.close(); saveSpy.mockRestore(); } }); it("close() stops the timer and is idempotent", async () => { const client = new MockClient(); const lib = await Library.open({ client, cacheDirectory, refreshIntervalSeconds: FAST_INTERVAL, }); const callsAfterOpen = client.collectionsSinceTimes.length; await lib.close(); await lib.close(); // second close must not throw expect(lib.status().closed).toBe(true); // No further refreshes fire once closed. await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 5)); expect(client.collectionsSinceTimes.length).toBe(callsAfterOpen); }); it("defaults cacheDirectory to the env-paths cache dir plus user id", async () => { const xdg = join(dir, "xdg-cache"); const prev = process.env.XDG_CACHE_HOME; process.env.XDG_CACHE_HOME = xdg; try { const client = new MockClient(); const lib = await Library.open({ client }); try { const expected = join( envPaths("quak", { suffix: "" }).cache, String(USER_ID), ); expect(lib.cacheDirectory).toBe(expected); expect(lib.cacheDirectory.startsWith(xdg)).toBe(true); expect(lib.cacheDirectory.endsWith(String(USER_ID))).toBe(true); } finally { await lib.close(); } } finally { if (prev === undefined) delete process.env.XDG_CACHE_HOME; else process.env.XDG_CACHE_HOME = prev; } }); });