On-disk content and thumbnail cache with per-photo fetch and prefetch (closes #46)

Adds the on-disk content and thumbnail cache keyed by fileID: originals/ and thumbnails/ under cacheDirectory, present-means-complete (streaming atomic rename), orphan temp reaping on open. Photo.original/thumbnail return a cached path with no network when present, else fetch through the shared request pool; thumbnails.ensure drives the thumbnail pool with priority, dedup and AbortSignal. One shared RequestPools set serves both the ML fetch and the content cache. Content-hash integrity is deferred (#68); authenticated streaming decrypt guarantees integrity now.

Model: opus-4-8

src/library/index.ts

@@ -41,6 +41,13 @@ import {
     type PhotosAPI,
     type TimelineAPI,
 } from "./read.js";
+import {
+    ContentCache,
+    type ContentSource,
+    type ThumbnailsAPI,
+    type EnsureOptions,
+    type EnsureResult,
+} from "./content.js";
 
 export {
     Album,
@@ -52,6 +59,18 @@ export {
     type TimelineGroup,
     type GroupBy,
 } from "./read.js";
+export {
+    type ContentSource,
+    type ContentResult,
+    type ContentEvent,
+    type ContentOptions,
+    type PhotoContent,
+    type ThumbnailsAPI,
+    type ThumbnailPriority,
+    type EnsureOptions,
+    type EnsureResult,
+    type EnsureEvent,
+} from "./content.js";
 import type { CollectionsPage, FilesPage } from "../client.js";
 import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js";
 import type { Collection, EnteFile } from "../model/types.js";
@@ -76,6 +95,11 @@ export interface LibraryClient {
         fileIDs: number[];
         fileKeys: Map<number, Uint8Array>;
     }): Promise<Map<number, MLData>>;
+    // The byte source for the on-disk content cache. Optional so a mock client
+    // that only serves metadata still satisfies the interface; when absent (and
+    // no explicit `contentSource` is passed to `open`) the content cache is
+    // disabled and `Photo.original`/`thumbnail` and `thumbnails.ensure` throw.
+    contentSource?(): ContentSource;
 }
 
 // A progress event for one unit of background work. A metadata "refresh" or an
@@ -96,14 +120,18 @@ export interface LibraryOptions {
     // Where `metadata.json` lives. Defaults to the env-paths cache directory
     // plus the user id, so each account has its own cache.
     cacheDirectory?: string;
-    // Persistent backup destination for later phases (backup, thumbnails); the
-    // refresh loop does not use it.
+    // Persistent backup destination. The refresh loop does not use it; the
+    // content cache treats an original already stored there as present.
     downloadDirectory?: string;
     refreshIntervalSeconds?: number;
     onProgress?: RefreshProgressCallback;
-    // The bounded request pools (issue #45). ML data is fetched through the
-    // metadata pool. Defaults to a fresh set at the design's caps.
+    // The bounded request pools (issue #45), shared by the ML-data fetch (the
+    // metadata pool) and the content cache. Defaults to a fresh set at the
+    // design's caps.
     pools?: RequestPools;
+    // Overrides the client's own `contentSource()`; mainly for tests that drive
+    // the cache with a stand-in source.
+    contentSource?: ContentSource;
 }
 
 export interface LibraryStatus {
@@ -138,9 +166,15 @@ export class Library {
     readonly albums: AlbumsAPI;
     readonly photos: PhotosAPI;
     readonly timeline: TimelineAPI;
+    // The thumbnail-prefetch surface (issue #46): drives the thumbnail pool
+    // with priority, dedup, and abort.
+    readonly thumbnails: ThumbnailsAPI;
 
     private readonly client: LibraryClient;
     private readonly store: MetadataStore;
+    // The on-disk content cache, or undefined when no content source is
+    // available (a metadata-only client with no explicit source).
+    private readonly cache?: ContentCache;
     private readonly userID: number;
     private readonly intervalMs: number;
     private readonly onProgress?: RefreshProgressCallback;
@@ -179,6 +213,7 @@ export class Library {
         onProgress?: RefreshProgressCallback;
         pools: RequestPools;
         mldata?: MLDataStore;
+        cache?: ContentCache;
     }) {
         this.client = args.client;
         this.store = args.store;
@@ -189,14 +224,27 @@ export class Library {
         this.onProgress = args.onProgress;
         this.pools = args.pools;
         this.mldata = args.mldata;
+        this.cache = args.cache;
         this.lastRecords = this.deriveNow();
 
         // The read namespaces derive fresh from the store on each call, so they
         // always reflect the latest refresh.
         const derive = (): DerivedRecords => this.deriveNow();
-        this.albums = makeAlbumsAPI(derive);
-        this.photos = makePhotosAPI(derive);
+        this.albums = makeAlbumsAPI(derive, this.cache);
+        this.photos = makePhotosAPI(derive, this.cache);
         this.timeline = makeTimelineAPI(derive);
+        this.thumbnails = {
+            ensure: (opts: EnsureOptions): Promise<EnsureResult[]> => {
+                if (!this.cache) {
+                    return Promise.reject(
+                        new Error(
+                            "thumbnails.ensure requires a library opened with a content cache",
+                        ),
+                    );
+                }
+                return this.cache.ensureThumbnails(opts);
+            },
+        };
     }
 
     // Load the cache and start the refresh loop. With an empty cache the first
@@ -218,12 +266,33 @@ export class Library {
             (opts.refreshIntervalSeconds ?? DEFAULT_REFRESH_INTERVAL_SECONDS) *
             1000;
 
+        // One request-pool set serves both the ML-data fetch and the content
+        // cache, so both honour the same concurrency caps.
+        const pools = opts.pools ?? new RequestPools();
+
         // The ML cache only earns its keep when the client can fetch ML data;
         // a client without that capability opens no `mldata/` directory.
         const mldata = opts.client.fetchMLData
             ? await MLDataStore.open(join(cacheDirectory, "mldata"))
             : undefined;
 
+        // Build the content cache from an explicit source or the client's own,
+        // and take its record of what is already cached (and reap orphan temp
+        // files) before the first projection, so cached paths are present from
+        // the start and the first refresh raises no spurious path-change diff.
+        const source = opts.contentSource ?? opts.client.contentSource?.();
+        let cache: ContentCache | undefined;
+        if (source) {
+            cache = new ContentCache({
+                pools,
+                source,
+                cacheDirectory,
+                downloadDirectory: opts.downloadDirectory,
+                getFile: (fileID) => store.getFileByID(fileID),
+            });
+            await cache.open();
+        }
+
         const lib = new Library({
             client: opts.client,
             store,
@@ -232,8 +301,9 @@ export class Library {
             downloadDirectory: opts.downloadDirectory,
             intervalMs,
             onProgress: opts.onProgress,
-            pools: opts.pools ?? new RequestPools(),
+            pools,
             mldata,
+            cache,
         });
 
         if (store.loadedFromDisk) {
@@ -522,12 +592,18 @@ export class Library {
         return [...byID.values()];
     }
 
-    // Gather every file membership and project the store into by-id records.
+    // Gather every file membership and project the store into by-id records,
+    // filling each record's cache paths from the content cache when present.
     private deriveNow(): DerivedRecords {
         const collections = this.store.listCollections();
         const files: EnteFile[] = [];
         for (const c of collections) files.push(...this.store.listFiles(c.id));
-        return deriveRecords(collections, files);
+        const cache = this.cache;
+        return deriveRecords(
+            collections,
+            files,
+            cache ? (fileID) => cache.pathsFor(fileID) : undefined,
+        );
     }
 
     private notify(change: LibraryChange): void {