Library.open with a transparent background refresh (closes #42)

Library.open loads metadata.json and serves reads from RAM: an empty cache awaits the first refresh, an existing cache returns at once and refreshes in the background so an unreachable server never stalls open(). A background timer refreshes every refreshIntervalSeconds (default 3), diffing only changed albums via the resumable cursor+tombstone enumerators and rewriting metadata.json only when something changed. A refresh failure is invisible to reads and surfaced via status()/onProgress; a failed save keeps status().lastError set and retries until one lands, so a stale disk is never masked. No sync()/refresh()/serverReachable surface; status() and close() included.

Model: opus-4-8

src/index.ts

@@ -33,6 +33,15 @@ export {
     requestEmailOTP,
     submitEmailOTP,
 } from "./auth/login.js";
+export {
+    Library,
+    DEFAULT_REFRESH_INTERVAL_SECONDS,
+    type LibraryClient,
+    type LibraryOptions,
+    type LibraryStatus,
+    type RefreshEvent,
+    type RefreshProgressCallback,
+} from "./library/index.js";
 export { decryptCollection, decryptFile } from "./model/index.js";
 export { downloadFile, downloadThumbnail } from "./download/index.js";
 export type {

src/library/index.ts

@@ -0,0 +1,316 @@
+// The library surface over the local cache.
+//
+// `Library.open()` loads the on-disk metadata store (issue #41), then starts
+// the refresh loop. When the cache loaded empty it awaits the first refresh,
+// so the library never opens onto an empty store it could have filled; when an
+// existing copy loaded, that first refresh runs in the background and `open()`
+// returns as soon as the cached data is ready to serve — a slow or unreachable
+// server no longer stalls opening. A background timer then refreshes every
+// `refreshIntervalSeconds`. Every read is answered from RAM — no read touches
+// the network. There is deliberately no `sync()`, no `refresh()`, no
+// `serverReachable` flag, and no "before each read" mode (design #36): the
+// only ways state changes are the refreshes above.
+//
+// A refresh stages all of its network work first and only mutates the store
+// once every fetch has succeeded. A refresh that fails partway therefore never
+// becomes visible to reads: the last good snapshot stays in place, and the
+// failure surfaces through `onProgress` and `status()` instead. A commit that
+// mutates RAM but then fails to persist keeps `status().lastError` set and the
+// store marked unsaved until a later save actually lands, so a stuck disk is
+// never masked by a subsequent empty refresh.
+
+import { join } from "node:path";
+import envPaths from "env-paths";
+
+import { MetadataStore } from "./store.js";
+import type { CollectionsPage, FilesPage } from "../client.js";
+import type { Collection, EnteFile } from "../model/types.js";
+
+export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3;
+
+// The slice of `Client` the library depends on. Narrowing to an interface lets
+// tests drive a mock with no crypto or network; the real `Client` satisfies it
+// structurally.
+export interface LibraryClient {
+    whoami(): { email: string; userID: number };
+    collectionsSince(args: { sinceTime: number }): Promise<CollectionsPage>;
+    filesSince(args: {
+        collectionID: number;
+        collectionKey: Uint8Array;
+        sinceTime: number;
+    }): Promise<FilesPage>;
+}
+
+// A single refresh cycle's progress. "started" fires before the network work,
+// then exactly one of "done" or "failed"; "failed" carries the error message.
+export interface RefreshEvent {
+    operation: "refresh";
+    status: "started" | "done" | "failed";
+    error?: string;
+}
+
+export type RefreshProgressCallback = (event: RefreshEvent) => void;
+
+export interface LibraryOptions {
+    client: LibraryClient;
+    // Where `metadata.json` lives. Defaults to the env-paths cache directory
+    // plus the user id, so each account has its own cache.
+    cacheDirectory?: string;
+    // Persistent backup destination for later phases (backup, thumbnails); the
+    // refresh loop does not use it.
+    downloadDirectory?: string;
+    refreshIntervalSeconds?: number;
+    onProgress?: RefreshProgressCallback;
+}
+
+export interface LibraryStatus {
+    userID: number;
+    collections: number;
+    files: number;
+    // Wall-clock ms of the last refresh that succeeded, or undefined if none
+    // has yet.
+    lastRefreshAt?: number;
+    // The message from the most recent refresh, set only while that refresh
+    // failed; cleared by the next success.
+    lastError?: string;
+    closed: boolean;
+}
+
+export class Library {
+    readonly cacheDirectory: string;
+    readonly downloadDirectory?: string;
+
+    private readonly client: LibraryClient;
+    private readonly store: MetadataStore;
+    private readonly userID: number;
+    private readonly intervalMs: number;
+    private readonly onProgress?: RefreshProgressCallback;
+
+    private timer?: ReturnType<typeof setTimeout>;
+    private refreshing = false;
+    private closed = false;
+    private lastRefreshAt?: number;
+    private lastError?: string;
+    // RAM holds changes disk has not yet accepted (an earlier save failed).
+    // Cleared only when a save actually succeeds; keeps the store trying to
+    // persist and the failure visible in `status()` until then.
+    private unsaved = false;
+
+    private constructor(args: {
+        client: LibraryClient;
+        store: MetadataStore;
+        userID: number;
+        cacheDirectory: string;
+        downloadDirectory?: string;
+        intervalMs: number;
+        onProgress?: RefreshProgressCallback;
+    }) {
+        this.client = args.client;
+        this.store = args.store;
+        this.userID = args.userID;
+        this.cacheDirectory = args.cacheDirectory;
+        this.downloadDirectory = args.downloadDirectory;
+        this.intervalMs = args.intervalMs;
+        this.onProgress = args.onProgress;
+    }
+
+    // Load the cache and start the refresh loop. With an empty cache the first
+    // refresh is awaited, so `open()` resolves onto populated data whenever the
+    // server is reachable; that awaited refresh may still fail, and the library
+    // then opens empty with the failure recorded in `status()`. With an
+    // existing cache the first refresh runs in the background and `open()`
+    // returns as soon as the cached data is ready — an unreachable server does
+    // not block opening.
+    static async open(opts: LibraryOptions): Promise<Library> {
+        const { userID } = opts.client.whoami();
+        const cacheDirectory =
+            opts.cacheDirectory ??
+            join(envPaths("quak", { suffix: "" }).cache, String(userID));
+        const store = await MetadataStore.load(
+            join(cacheDirectory, "metadata.json"),
+        );
+        const intervalMs =
+            (opts.refreshIntervalSeconds ?? DEFAULT_REFRESH_INTERVAL_SECONDS) *
+            1000;
+
+        const lib = new Library({
+            client: opts.client,
+            store,
+            userID,
+            cacheDirectory,
+            downloadDirectory: opts.downloadDirectory,
+            intervalMs,
+            onProgress: opts.onProgress,
+        });
+
+        if (store.loadedFromDisk) {
+            // An existing copy already answers reads; refresh in the background
+            // and start the interval once that first cycle settles.
+            void lib.runRefresh().then(() => lib.scheduleNext());
+        } else {
+            // Nothing was cached: wait for the first refresh to fill the store
+            // (or fail) rather than resolve onto an empty library.
+            await lib.runRefresh();
+            lib.scheduleNext();
+        }
+        return lib;
+    }
+
+    listCollections(): Collection[] {
+        return this.store.listCollections();
+    }
+
+    getCollection(id: number): Collection | undefined {
+        return this.store.getCollection(id);
+    }
+
+    listFiles(collectionID: number): EnteFile[] {
+        return this.store.listFiles(collectionID);
+    }
+
+    getFile(collectionID: number, fileID: number): EnteFile | undefined {
+        return this.store.getFile(collectionID, fileID);
+    }
+
+    status(): LibraryStatus {
+        let files = 0;
+        const collections = this.store.listCollections();
+        for (const c of collections) {
+            files += this.store.listFiles(c.id).length;
+        }
+        return {
+            userID: this.store.userID,
+            collections: collections.length,
+            files,
+            lastRefreshAt: this.lastRefreshAt,
+            lastError: this.lastError,
+            closed: this.closed,
+        };
+    }
+
+    // Stop the background timer. Idempotent. An in-flight refresh is left to
+    // finish; it will not schedule another cycle once closed.
+    close(): void {
+        this.closed = true;
+        if (this.timer !== undefined) {
+            clearTimeout(this.timer);
+            this.timer = undefined;
+        }
+    }
+
+    private scheduleNext(): void {
+        if (this.closed) return;
+        this.timer = setTimeout(() => {
+            void this.runRefresh().then(() => this.scheduleNext());
+        }, this.intervalMs);
+        // Do not keep the process alive for the sake of the timer.
+        this.timer.unref?.();
+    }
+
+    // One refresh cycle, guarded so a failure never escapes and overlapping
+    // cycles never run. Errors are reported, not thrown.
+    private async runRefresh(): Promise<void> {
+        if (this.closed || this.refreshing) return;
+        this.refreshing = true;
+        this.emit({ operation: "refresh", status: "started" });
+        try {
+            await this.refreshOnce();
+            this.lastRefreshAt = Date.now();
+            this.lastError = undefined;
+            this.emit({ operation: "refresh", status: "done" });
+        } catch (err) {
+            const error = err instanceof Error ? err.message : String(err);
+            this.lastError = error;
+            this.emit({ operation: "refresh", status: "failed", error });
+        } finally {
+            this.refreshing = false;
+        }
+    }
+
+    // Fetch every change since the stored cursor, then commit. All network
+    // reads happen before any store mutation, so a fetch that throws leaves the
+    // store untouched and the previous snapshot intact.
+    private async refreshOnce(): Promise<void> {
+        const page = await this.client.collectionsSince({
+            sinceTime: this.store.collectionsSinceTime,
+        });
+
+        // Stage per-collection file diffs. A collection's files are
+        // re-enumerated only when its updationTime has advanced past the cached
+        // copy; an unchanged album's file list cannot have changed. New
+        // collections enumerate from the beginning of time.
+        const filePages: { collectionID: number; page: FilesPage }[] = [];
+        for (const collection of page.collections) {
+            const known = this.store.getCollection(collection.id);
+            if (known && collection.updationTime <= known.updationTime)
+                continue;
+            const filePage = await this.client.filesSince({
+                collectionID: collection.id,
+                collectionKey: collection.key,
+                sinceTime: known ? known.updationTime : 0,
+            });
+            filePages.push({ collectionID: collection.id, page: filePage });
+        }
+
+        // Network work done; commit to the store and persist only if something
+        // actually changed.
+        let changed = false;
+
+        if (this.store.userID !== this.userID) {
+            this.store.userID = this.userID;
+            changed = true;
+        }
+
+        for (const id of page.deleted) {
+            if (this.store.getCollection(id)) {
+                this.store.deleteCollection(id);
+                changed = true;
+            }
+        }
+        for (const collection of page.collections) {
+            this.store.putCollection(collection);
+            changed = true;
+        }
+
+        for (const { collectionID, page: filePage } of filePages) {
+            for (const id of filePage.deleted) {
+                if (this.store.getFile(collectionID, id)) {
+                    this.store.deleteFile(collectionID, id);
+                    changed = true;
+                }
+            }
+            for (const f of filePage.files) {
+                this.store.putFile(f);
+                changed = true;
+            }
+        }
+
+        if (page.cursor !== this.store.collectionsSinceTime) {
+            this.store.collectionsSinceTime = page.cursor;
+            changed = true;
+        }
+
+        if (changed) this.unsaved = true;
+
+        // Persist whenever RAM holds changes disk has not accepted — including
+        // changes an earlier cycle staged whose save failed. `unsaved` clears
+        // only once a save lands, so a save failure both stays visible through
+        // `status().lastError` (the throw below records it) and keeps being
+        // retried, instead of a later empty refresh silently clearing it while
+        // the on-disk cache is still behind RAM.
+        if (this.unsaved) {
+            await this.store.save();
+            this.unsaved = false;
+        }
+    }
+
+    private emit(event: RefreshEvent): void {
+        if (!this.onProgress) return;
+        // A misbehaving callback must not break the refresh loop.
+        try {
+            this.onProgress(event);
+        } catch {
+            // ignore
+        }
+    }
+}

src/library/store.ts

@@ -58,6 +58,12 @@ export class MetadataStore {
     userID = 0;
     collectionsSinceTime: Microseconds = 0;
 
+    // True when `load` populated this store from a valid existing file; false
+    // on a first run or a missing/corrupt/wrong-version file that loaded empty.
+    // `Library.open` reads it to decide whether the first refresh may run in
+    // the background (an existing copy already serves reads) or must be awaited.
+    loadedFromDisk = false;
+
     private readonly collections = new Map<number, Collection>();
     private readonly files = new Map<string, EnteFile>();
 
@@ -83,6 +89,7 @@ export class MetadataStore {
             if (parsed.schemaVersion !== METADATA_SCHEMA_VERSION) {
                 return store;
             }
+            store.loadedFromDisk = true;
             store.userID = parsed.userID ?? 0;
             store.collectionsSinceTime = parsed.collectionsSinceTime ?? 0;
             for (const stored of parsed.collections ?? []) {