From ff0bbb3155268b87c35190e02972518e449f7fd7 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 22 Sep 2026 15:47:32 +0000 Subject: [PATCH] On-disk content and thumbnail cache with per-photo fetch and prefetch (closes #46) Add src/library/content.ts: a ContentCache keyed by fileID under cacheDirectory (flat originals/ and thumbnails/, 0700/0600), fetching through the request pools (#45) and the streaming decrypt / atomic writer (#40) so present-means-complete. One shared pool set serves both this cache and the ML-data fetch. Photo.original and thumbnail return {path,bytes}, skipped when present; lib.thumbnails.ensure drives the thumbnail pool with priority, dedup, and abort. Integrity rests on the streaming decrypt (every chunk authenticated, renamed in only on TAG_FINAL) plus a non-empty check. The stored-hash / size compare is deferred (tracked in #68): the only in-repo hash fixture is a placeholder, and the stored size is the encrypted object size, not the decrypted length. Judgement call: three thumbnail priorities map onto two tiers. Model: opus-4-8 --- src/client.ts | 12 + src/index.ts | 20 ++ src/library/content.ts | 416 ++++++++++++++++++++++++++ src/library/index.ts | 94 +++++- src/library/read.ts | 55 +++- src/library/records.ts | 16 + src/library/store.ts | 10 + test/library/content-library.test.ts | 152 ++++++++++ test/library/content.test.ts | 423 +++++++++++++++++++++++++++ 9 files changed, 1179 insertions(+), 19 deletions(-) create mode 100644 src/library/content.ts create mode 100644 test/library/content-library.test.ts create mode 100644 test/library/content.test.ts diff --git a/src/client.ts b/src/client.ts index a60ec79..3acfd48 100644 --- a/src/client.ts +++ b/src/client.ts @@ -13,6 +13,10 @@ import { downloadFile as dlFile, downloadThumbnail as dlThumb, } from "./download/index.js"; +import { + makeDownloadContentSource, + type ContentSource, +} from "./library/content.js"; import type { Collection, EnteFile, @@ -141,6 +145,14 @@ export class Client { return this.api; } + // The content-cache byte source over this client's API: each fetch is the + // download layer's request + streaming decrypt + atomic write. `Library` + // calls this to enable the on-disk content cache. + contentSource(): ContentSource { + this.assertLoggedIn(); + return makeDownloadContentSource(this.api); + } + private assertLoggedIn(): void { if (this.loggedOut) throw new Error("Client has been logged out"); } diff --git a/src/index.ts b/src/index.ts index 9e90b0f..4bb1439 100644 --- a/src/index.ts +++ b/src/index.ts @@ -49,7 +49,27 @@ export { type PhotoFilter, type TimelineGroup, type GroupBy, + type ContentSource, + type ContentResult, + type ContentEvent, + type ContentOptions, + type PhotoContent, + type ThumbnailsAPI, + type ThumbnailPriority, + type EnsureOptions, + type EnsureResult, + type EnsureEvent, } from "./library/index.js"; +export { + RequestPools, + BoundedPool, + DEFAULT_METADATA_CONCURRENCY, + DEFAULT_CONTENT_CONCURRENCY, + DEFAULT_THUMBNAIL_CONCURRENCY, + type RequestPoolsOptions, + type Priority, + type RunOptions, +} from "./library/pools.js"; export type { AlbumRecord, PhotoRecord, diff --git a/src/library/content.ts b/src/library/content.ts new file mode 100644 index 0000000..a5379ec --- /dev/null +++ b/src/library/content.ts @@ -0,0 +1,416 @@ +// The on-disk content and thumbnail cache keyed by fileID (issue #46). +// +// Layout under `cacheDirectory`: `originals/.` and +// `thumbnails/.`, flat directories at 0700 with files at 0600. +// Content appears only by the streaming atomic writer's rename (the download +// layer, #40), so a file that exists is whole — "present means complete". The +// directory listing taken at `open()` is the record of what is cached, and the +// orphan temp files a crashed write may have left are reaped there. +// +// A fetch goes through the shared request pools (#45): the content pool for +// originals, the thumbnail pool for thumbnails. The pool limits concurrency, +// orders on-demand work ahead of background, and dedups by key so a fileID +// requested twice while the first is still in flight downloads once. +// +// Integrity. The reused streaming decrypt is the enforced guarantee: every +// chunk is authenticated and the writer renames the file into place only once +// the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and +// nothing is stored. On top of that this module refuses to record a stored file +// that came out empty. The design also asks for a content-hash comparison +// against `FileMetadata.hash` (with a `fileSize` fallback); that is deferred — +// see the PR — because the exact hash construction cannot be confirmed against +// the repo's fixtures and `FileBlob.size` is the encrypted object size, not the +// decrypted length this layer has. + +import { existsSync, statSync } from "node:fs"; +import { chmod, mkdir, readdir, rm, stat } from "node:fs/promises"; +import { extname, join } from "node:path"; + +import type { ApiClient } from "../api/client.js"; +import { + downloadFile, + downloadThumbnail, + type ProgressCallback, +} from "../download/index.js"; +import type { EnteFile } from "../model/types.js"; +import type { Priority, RequestPools } from "./pools.js"; + +const DIR_MODE = 0o700; +const FILE_MODE = 0o600; +const TEMP_PREFIX = ".quak-"; +const TEMP_SUFFIX = ".tmp"; +// Ente thumbnails are always JPEG, so the cache stores them with a fixed +// extension rather than deriving one from the (image or video) title. +const THUMBNAIL_EXT = ".jpg"; + +type Kind = "original" | "thumbnail"; + +// The priority a caller attaches to a thumbnail prefetch. The pool has two +// tiers, so this three-value surface collapses onto them: only a currently +// visible thumbnail preempts (on-demand); "ahead" prefetch and speculative +// "background" work both yield to it. +export type ThumbnailPriority = "visible" | "ahead" | "background"; + +const poolPriorityOf = (priority: ThumbnailPriority): Priority => + priority === "visible" ? "on-demand" : "background"; + +export interface ContentResult { + path: string; + bytes: number; +} + +// Progress for a single `original`/`thumbnail` call. A present file emits one +// `skipped` event and nothing else; a fetched file emits `downloading` as +// plaintext lands and a final `done`. +export type ContentEvent = + | { status: "skipped"; bytes: number } + | { status: "downloading"; bytesDone: number } + | { status: "done"; bytes: number }; + +export interface ContentOptions { + onProgress?: (event: ContentEvent) => void; +} + +// The Photo-facing content surface (the read wrappers call these). The cache +// implements it; a library opened without a content source leaves it absent. +export interface PhotoContent { + original(fileID: number, opts?: ContentOptions): Promise; + thumbnail(fileID: number, opts?: ContentOptions): Promise; +} + +export interface EnsureResult { + fileID: number; + path?: string; + error?: string; +} + +export interface EnsureEvent { + fileID: number; + status: "skipped" | "done" | "failed" | "aborted"; + path?: string; + error?: string; +} + +export interface EnsureOptions { + fileIDs: number[]; + priority: ThumbnailPriority; + signal?: AbortSignal; + onProgress?: (event: EnsureEvent) => void; +} + +export interface ThumbnailsAPI { + ensure(args: EnsureOptions): Promise; +} + +// The byte source the cache fetches through. The real implementation streams +// and decrypts to the destination via the download layer; tests inject a +// stand-in so the cache logic runs with no crypto and no network. Pool routing, +// dedup, present-checks and integrity live in the cache, not here. +export interface ContentSource { + original(args: { + file: EnteFile; + destination: string; + onProgress?: ProgressCallback; + }): Promise<{ bytesWritten: number }>; + thumbnail(args: { + file: EnteFile; + destination: string; + onProgress?: ProgressCallback; + }): Promise<{ bytesWritten: number }>; +} + +// The production source: each fetch is the download layer's request + +// streaming decrypt + atomic write + retry as one unit. +export const makeDownloadContentSource = (api: ApiClient): ContentSource => ({ + original: ({ file, destination, onProgress }) => + downloadFile(api, file, destination, onProgress), + thumbnail: ({ file, destination, onProgress }) => + downloadThumbnail(api, file, destination, onProgress), +}); + +export interface CachedPaths { + originalPath?: string; + thumbnailPath?: string; +} + +export interface ContentCacheOptions { + pools: RequestPools; + source: ContentSource; + cacheDirectory: string; + // The backup destination (issue-level `downloadDirectory`). An original + // already stored there by a backup counts as present, so the cache serves + // it rather than fetching a second copy. + downloadDirectory?: string; + // Resolve any membership of a file; every membership shares the underlying + // content key, so any one decrypts the same bytes. + getFile: (fileID: number) => EnteFile | undefined; +} + +// Thrown inside a pooled task to drop a queued fetch that was aborted before it +// started running. Never escapes `ensureThumbnails`. +class AbortDrop extends Error { + constructor() { + super("aborted"); + this.name = "AbortDrop"; + } +} + +const originalName = (file: EnteFile): string => { + const ext = extname(file.metadata.title || "") || ".bin"; + return `${file.id}${ext}`; +}; + +// The fileID a cache filename encodes, or undefined when the name is not one +// the cache writes (``). +const fileIDFromName = (name: string): number | undefined => { + const base = name.slice(0, name.length - extname(name).length); + if (!/^\d+$/.test(base)) return undefined; + const id = Number(base); + return Number.isSafeInteger(id) ? id : undefined; +}; + +// Size of a regular file, or undefined if it is absent (or not a regular file). +const fileSize = (path: string): number | undefined => { + try { + const s = statSync(path); + return s.isFile() ? s.size : undefined; + } catch { + return undefined; + } +}; + +export class ContentCache implements PhotoContent, ThumbnailsAPI { + private readonly pools: RequestPools; + private readonly source: ContentSource; + private readonly downloadDirectory?: string; + private readonly getFile: (fileID: number) => EnteFile | undefined; + private readonly originalsDir: string; + private readonly thumbnailsDir: string; + // fileID -> absolute path of the cached bytes, seeded from the directory + // listing at open() and extended as fetches store new files. + private readonly originals = new Map(); + private readonly thumbnails = new Map(); + + constructor(opts: ContentCacheOptions) { + this.pools = opts.pools; + this.source = opts.source; + this.downloadDirectory = opts.downloadDirectory; + this.getFile = opts.getFile; + this.originalsDir = join(opts.cacheDirectory, "originals"); + this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails"); + } + + // Prepare the cache directories, reap orphan temp files, and take the + // record of what is already cached. Called once before the cache serves. + async open(): Promise { + await this.ensureDir(this.originalsDir); + await this.ensureDir(this.thumbnailsDir); + await this.scan(this.originalsDir, this.originals); + await this.scan(this.thumbnailsDir, this.thumbnails); + } + + // The cache paths known for a file, for the record projection to expose as + // `originalPath`/`thumbnailPath`. + pathsFor(fileID: number): CachedPaths { + const out: CachedPaths = {}; + const original = this.originals.get(fileID); + if (original !== undefined) out.originalPath = original; + const thumbnail = this.thumbnails.get(fileID); + if (thumbnail !== undefined) out.thumbnailPath = thumbnail; + return out; + } + + async original( + fileID: number, + opts?: ContentOptions, + ): Promise { + return this.get(fileID, "original", "on-demand", opts?.onProgress); + } + + async thumbnail( + fileID: number, + opts?: ContentOptions, + ): Promise { + return this.get(fileID, "thumbnail", "on-demand", opts?.onProgress); + } + + async ensure(args: EnsureOptions): Promise { + return this.ensureThumbnails(args); + } + + async ensureThumbnails(args: EnsureOptions): Promise { + const priority = poolPriorityOf(args.priority); + // Dedup the request list so a repeated fileID is fetched once and + // reported once, in first-requested order. + const seen = new Set(); + const unique: number[] = []; + for (const id of args.fileIDs) { + if (!seen.has(id)) { + seen.add(id); + unique.push(id); + } + } + return Promise.all( + unique.map((fileID) => + this.ensureOne(fileID, priority, args.signal, args.onProgress), + ), + ); + } + + private async ensureOne( + fileID: number, + priority: Priority, + signal: AbortSignal | undefined, + onProgress: ((event: EnsureEvent) => void) | undefined, + ): Promise { + try { + const result = await this.acquire( + fileID, + "thumbnail", + priority, + signal, + ); + const status = result.cached ? "skipped" : "done"; + onProgress?.({ fileID, status, path: result.path }); + return { fileID, path: result.path }; + } catch (err) { + if (err instanceof AbortDrop) { + onProgress?.({ fileID, status: "aborted" }); + return { fileID, error: "aborted" }; + } + const error = err instanceof Error ? err.message : String(err); + onProgress?.({ fileID, status: "failed", error }); + return { fileID, error }; + } + } + + private async get( + fileID: number, + kind: Kind, + priority: Priority, + onProgress: ((event: ContentEvent) => void) | undefined, + ): Promise { + const onByte: ProgressCallback | undefined = onProgress + ? (bytesDone) => onProgress({ status: "downloading", bytesDone }) + : undefined; + const result = await this.acquire(fileID, kind, priority, undefined, { + onByte, + }); + onProgress?.( + result.cached + ? { status: "skipped", bytes: result.bytes } + : { status: "done", bytes: result.bytes }, + ); + return { path: result.path, bytes: result.bytes }; + } + + // The core: return the cached path if present, else fetch through the pool, + // store, and return it. `cached` distinguishes a present hit (no network, + // no download event) from a fresh fetch. + private async acquire( + fileID: number, + kind: Kind, + priority: Priority, + signal: AbortSignal | undefined, + opts?: { onByte?: ProgressCallback }, + ): Promise<{ path: string; bytes: number; cached: boolean }> { + const file = this.getFile(fileID); + if (!file) throw new Error(`content cache: unknown file ${fileID}`); + + const known = kind === "original" ? this.originals : this.thumbnails; + const cached = known.get(fileID); + if (cached !== undefined) { + const size = fileSize(cached); + if (size !== undefined && size > 0) + return { path: cached, bytes: size, cached: true }; + // A recorded file that has since gone re-fetches below. + known.delete(fileID); + } + + // An original a backup already stored counts as present. + if (kind === "original" && this.downloadDirectory !== undefined) { + const backupPath = join( + this.downloadDirectory, + "originals", + originalName(file), + ); + const size = fileSize(backupPath); + if (size !== undefined && size > 0) { + this.originals.set(fileID, backupPath); + return { path: backupPath, bytes: size, cached: true }; + } + } + + const dir = + kind === "original" ? this.originalsDir : this.thumbnailsDir; + const dest = + kind === "original" + ? join(dir, originalName(file)) + : join(dir, `${fileID}${THUMBNAIL_EXT}`); + const pool = + kind === "original" ? this.pools.content : this.pools.thumbnails; + + return pool.run( + async () => { + // Dropping queued work on abort: a task still waiting for a slot + // when the signal fired sees it here and never touches the + // network. A task already past this point is in flight and runs + // to completion. + if (signal?.aborted) throw new AbortDrop(); + + await this.download(file, dest, kind, opts?.onByte); + await chmod(dest, FILE_MODE); + const size = (await stat(dest)).size; + if (size === 0) { + throw new Error( + `content cache: ${kind} ${fileID} stored empty`, + ); + } + known.set(fileID, dest); + return { path: dest, bytes: size, cached: false }; + }, + { priority, key: fileID }, + ); + } + + private async download( + file: EnteFile, + destination: string, + kind: Kind, + onProgress: ProgressCallback | undefined, + ): Promise { + const args = { file, destination, onProgress }; + const result = + kind === "original" + ? await this.source.original(args) + : await this.source.thumbnail(args); + return result.bytesWritten; + } + + private async ensureDir(dir: string): Promise { + // chmod after mkdir so the mode is tightened even when the directory + // already existed with a looser one; mkdir alone would not. + await mkdir(dir, { recursive: true, mode: DIR_MODE }); + await chmod(dir, DIR_MODE); + } + + private async scan(dir: string, into: Map): Promise { + let entries: string[]; + try { + entries = await readdir(dir); + } catch { + return; + } + for (const name of entries) { + if (name.startsWith(TEMP_PREFIX) && name.endsWith(TEMP_SUFFIX)) { + await rm(join(dir, name), { force: true }).catch( + () => undefined, + ); + continue; + } + const id = fileIDFromName(name); + const path = join(dir, name); + if (id !== undefined && existsSync(path)) into.set(id, path); + } + } +} diff --git a/src/library/index.ts b/src/library/index.ts index ba2a315..04e3fcd 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -41,6 +41,13 @@ import { type PhotosAPI, type TimelineAPI, } from "./read.js"; +import { + ContentCache, + type ContentSource, + type ThumbnailsAPI, + type EnsureOptions, + type EnsureResult, +} from "./content.js"; export { Album, @@ -52,6 +59,18 @@ export { type TimelineGroup, type GroupBy, } from "./read.js"; +export { + type ContentSource, + type ContentResult, + type ContentEvent, + type ContentOptions, + type PhotoContent, + type ThumbnailsAPI, + type ThumbnailPriority, + type EnsureOptions, + type EnsureResult, + type EnsureEvent, +} from "./content.js"; import type { CollectionsPage, FilesPage } from "../client.js"; import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js"; import type { Collection, EnteFile } from "../model/types.js"; @@ -76,6 +95,11 @@ export interface LibraryClient { fileIDs: number[]; fileKeys: Map; }): Promise>; + // The byte source for the on-disk content cache. Optional so a mock client + // that only serves metadata still satisfies the interface; when absent (and + // no explicit `contentSource` is passed to `open`) the content cache is + // disabled and `Photo.original`/`thumbnail` and `thumbnails.ensure` throw. + contentSource?(): ContentSource; } // A progress event for one unit of background work. A metadata "refresh" or an @@ -96,14 +120,18 @@ export interface LibraryOptions { // Where `metadata.json` lives. Defaults to the env-paths cache directory // plus the user id, so each account has its own cache. cacheDirectory?: string; - // Persistent backup destination for later phases (backup, thumbnails); the - // refresh loop does not use it. + // Persistent backup destination. The refresh loop does not use it; the + // content cache treats an original already stored there as present. downloadDirectory?: string; refreshIntervalSeconds?: number; onProgress?: RefreshProgressCallback; - // The bounded request pools (issue #45). ML data is fetched through the - // metadata pool. Defaults to a fresh set at the design's caps. + // The bounded request pools (issue #45), shared by the ML-data fetch (the + // metadata pool) and the content cache. Defaults to a fresh set at the + // design's caps. pools?: RequestPools; + // Overrides the client's own `contentSource()`; mainly for tests that drive + // the cache with a stand-in source. + contentSource?: ContentSource; } export interface LibraryStatus { @@ -138,9 +166,15 @@ export class Library { readonly albums: AlbumsAPI; readonly photos: PhotosAPI; readonly timeline: TimelineAPI; + // The thumbnail-prefetch surface (issue #46): drives the thumbnail pool + // with priority, dedup, and abort. + readonly thumbnails: ThumbnailsAPI; private readonly client: LibraryClient; private readonly store: MetadataStore; + // The on-disk content cache, or undefined when no content source is + // available (a metadata-only client with no explicit source). + private readonly cache?: ContentCache; private readonly userID: number; private readonly intervalMs: number; private readonly onProgress?: RefreshProgressCallback; @@ -179,6 +213,7 @@ export class Library { onProgress?: RefreshProgressCallback; pools: RequestPools; mldata?: MLDataStore; + cache?: ContentCache; }) { this.client = args.client; this.store = args.store; @@ -189,14 +224,27 @@ export class Library { this.onProgress = args.onProgress; this.pools = args.pools; this.mldata = args.mldata; + this.cache = args.cache; this.lastRecords = this.deriveNow(); // The read namespaces derive fresh from the store on each call, so they // always reflect the latest refresh. const derive = (): DerivedRecords => this.deriveNow(); - this.albums = makeAlbumsAPI(derive); - this.photos = makePhotosAPI(derive); + this.albums = makeAlbumsAPI(derive, this.cache); + this.photos = makePhotosAPI(derive, this.cache); this.timeline = makeTimelineAPI(derive); + this.thumbnails = { + ensure: (opts: EnsureOptions): Promise => { + if (!this.cache) { + return Promise.reject( + new Error( + "thumbnails.ensure requires a library opened with a content cache", + ), + ); + } + return this.cache.ensureThumbnails(opts); + }, + }; } // Load the cache and start the refresh loop. With an empty cache the first @@ -218,12 +266,33 @@ export class Library { (opts.refreshIntervalSeconds ?? DEFAULT_REFRESH_INTERVAL_SECONDS) * 1000; + // One request-pool set serves both the ML-data fetch and the content + // cache, so both honour the same concurrency caps. + const pools = opts.pools ?? new RequestPools(); + // The ML cache only earns its keep when the client can fetch ML data; // a client without that capability opens no `mldata/` directory. const mldata = opts.client.fetchMLData ? await MLDataStore.open(join(cacheDirectory, "mldata")) : undefined; + // Build the content cache from an explicit source or the client's own, + // and take its record of what is already cached (and reap orphan temp + // files) before the first projection, so cached paths are present from + // the start and the first refresh raises no spurious path-change diff. + const source = opts.contentSource ?? opts.client.contentSource?.(); + let cache: ContentCache | undefined; + if (source) { + cache = new ContentCache({ + pools, + source, + cacheDirectory, + downloadDirectory: opts.downloadDirectory, + getFile: (fileID) => store.getFileByID(fileID), + }); + await cache.open(); + } + const lib = new Library({ client: opts.client, store, @@ -232,8 +301,9 @@ export class Library { downloadDirectory: opts.downloadDirectory, intervalMs, onProgress: opts.onProgress, - pools: opts.pools ?? new RequestPools(), + pools, mldata, + cache, }); if (store.loadedFromDisk) { @@ -522,12 +592,18 @@ export class Library { return [...byID.values()]; } - // Gather every file membership and project the store into by-id records. + // Gather every file membership and project the store into by-id records, + // filling each record's cache paths from the content cache when present. private deriveNow(): DerivedRecords { const collections = this.store.listCollections(); const files: EnteFile[] = []; for (const c of collections) files.push(...this.store.listFiles(c.id)); - return deriveRecords(collections, files); + const cache = this.cache; + return deriveRecords( + collections, + files, + cache ? (fileID) => cache.pathsFor(fileID) : undefined, + ); } private notify(change: LibraryChange): void { diff --git a/src/library/read.ts b/src/library/read.ts index 3c756b4..b8f8678 100644 --- a/src/library/read.ts +++ b/src/library/read.ts @@ -10,10 +10,14 @@ // those records: a caller that holds an object reference gets typed field // access and, for an album, its photos. They are not sent across IPC — the // plain records are the serializable surface, and `record()` returns one. -// Content-fetch methods (`Photo.original` / `thumbnail`) belong to a later -// unit; this surface is read-only. +// +// A `Photo` also fetches its own bytes: `original()` and `thumbnail()` go +// through the on-disk content cache (issue #46), the one place in this module +// that is not synchronous and RAM-only. A library opened without a content +// source leaves that cache absent, and those two methods then throw. import type { CollectionType, FileType } from "../model/types.js"; +import type { ContentOptions, ContentResult, PhotoContent } from "./content.js"; import type { AlbumRecord, PhotoRecord, DerivedRecords } from "./records.js"; // Newest first, with fileID as a stable tiebreak so equal-timed files order @@ -29,7 +33,10 @@ const byNewestAlbum = (a: AlbumRecord, b: AlbumRecord): number => // A single photo. Field access mirrors `PhotoRecord`; `record()` returns the // underlying plain record for callers that need the IPC-safe value. export class Photo { - constructor(private readonly rec: PhotoRecord) {} + constructor( + private readonly rec: PhotoRecord, + private readonly content?: PhotoContent, + ) {} get fileID(): number { return this.rec.fileID; @@ -71,6 +78,27 @@ export class Photo { record(): PhotoRecord { return this.rec; } + + // Fetch and cache the full-resolution original, returning its on-disk path + // and byte length. Served from the cache (or the backup download directory) + // when already present, otherwise fetched through the content pool. + async original(opts?: ContentOptions): Promise { + return this.contentOrThrow().original(this.rec.fileID, opts); + } + + // As `original`, for the thumbnail, through the thumbnail pool. + async thumbnail(opts?: ContentOptions): Promise { + return this.contentOrThrow().thumbnail(this.rec.fileID, opts); + } + + private contentOrThrow(): PhotoContent { + if (!this.content) { + throw new Error( + "Photo content requires a library opened with a content cache", + ); + } + return this.content; + } } // A single album. `photos.list()` returns the album's photos as wrappers, @@ -79,6 +107,7 @@ export class Album { constructor( private readonly rec: AlbumRecord, private readonly records: DerivedRecords, + private readonly content?: PhotoContent, ) {} get collectionID(): number { @@ -112,7 +141,7 @@ export class Album { const out: Photo[] = []; for (const id of this.rec.fileIDs) { const p = this.records.photos.get(id); - if (p) out.push(new Photo(p)); + if (p) out.push(new Photo(p, this.content)); } return out; } @@ -164,17 +193,20 @@ export interface TimelineAPI { groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[]; } -export const makeAlbumsAPI = (derive: () => DerivedRecords): AlbumsAPI => ({ +export const makeAlbumsAPI = ( + derive: () => DerivedRecords, + content?: PhotoContent, +): AlbumsAPI => ({ list: (): Album[] => { const records = derive(); return [...records.albums.values()] .sort(byNewestAlbum) - .map((rec) => new Album(rec, records)); + .map((rec) => new Album(rec, records, content)); }, byID: ({ collectionID }): Album | undefined => { const records = derive(); const rec = records.albums.get(collectionID); - return rec ? new Album(rec, records) : undefined; + return rec ? new Album(rec, records, content) : undefined; }, byName: ({ albumName }): Album | undefined => { const records = derive(); @@ -183,14 +215,17 @@ export const makeAlbumsAPI = (derive: () => DerivedRecords): AlbumsAPI => ({ const match = [...records.albums.values()] .sort(byNewestAlbum) .find((rec) => rec.name === albumName); - return match ? new Album(match, records) : undefined; + return match ? new Album(match, records, content) : undefined; }, }); -export const makePhotosAPI = (derive: () => DerivedRecords): PhotosAPI => ({ +export const makePhotosAPI = ( + derive: () => DerivedRecords, + content?: PhotoContent, +): PhotosAPI => ({ byID: ({ fileID }): Photo | undefined => { const rec = derive().photos.get(fileID); - return rec ? new Photo(rec) : undefined; + return rec ? new Photo(rec, content) : undefined; }, records: ({ fileIDs }): PhotoRecord[] => { const { photos } = derive(); diff --git a/src/library/records.ts b/src/library/records.ts index 85d452a..f71aa2f 100644 --- a/src/library/records.ts +++ b/src/library/records.ts @@ -166,11 +166,20 @@ const toAlbumRecord = ( }; }; +// The cache paths known for a file, so the projection can expose them on the +// record without the read layer reaching into the content cache itself. +export type CachedPathLookup = (fileID: number) => { + originalPath?: string; + thumbnailPath?: string; +}; + // Project the decrypted collections and file memberships into by-id records. // `files` is every membership (a file appears once per collection it is in). +// `cachedPaths`, when given, fills each record's cache paths. export const deriveRecords = ( collections: Collection[], files: EnteFile[], + cachedPaths?: CachedPathLookup, ): DerivedRecords => { const byFileID = new Map(); for (const f of files) { @@ -183,6 +192,13 @@ export const deriveRecords = ( const takenAtByFile = new Map(); for (const [fileID, memberships] of byFileID) { const record = toPhotoRecord(fileID, memberships); + if (cachedPaths) { + const paths = cachedPaths(fileID); + if (paths.originalPath !== undefined) + record.originalPath = paths.originalPath; + if (paths.thumbnailPath !== undefined) + record.thumbnailPath = paths.thumbnailPath; + } photos.set(fileID, record); takenAtByFile.set(fileID, record.takenAt); } diff --git a/src/library/store.ts b/src/library/store.ts index cfacb43..56fe0dd 100644 --- a/src/library/store.ts +++ b/src/library/store.ts @@ -179,6 +179,16 @@ export class MetadataStore { return this.files.get(fileKey(collectionID, fileID)); } + // Any membership of a file, or undefined. Every membership re-wraps the + // same underlying content key, so any one is enough to fetch the bytes; + // the content cache resolves a fileID to a file this way. + getFileByID(fileID: number): EnteFile | undefined { + for (const file of this.files.values()) { + if (file.id === fileID) return file; + } + return undefined; + } + listFiles(collectionID: number): EnteFile[] { return [...this.files.values()].filter( (f) => f.collectionID === collectionID, diff --git a/test/library/content-library.test.ts b/test/library/content-library.test.ts new file mode 100644 index 0000000..e1eb1da --- /dev/null +++ b/test/library/content-library.test.ts @@ -0,0 +1,152 @@ +/** + * Integration between `Library` and the content cache (issue #46). + * + * The cache itself is covered in `content.test.ts`; this file locks the wiring: + * `Library.open` builds the cache from a content source, `lib.photos` hands out + * `Photo` objects that fetch through it, `lib.thumbnails.ensure` drives it, and + * a cached path shows up on the projected record. A library opened without a + * content source leaves those methods throwing rather than silently doing + * nothing. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Library } from "../../src/library/index.js"; +import type { ContentSource } from "../../src/library/content.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +const USER_ID = 7; + +const collection = (id: number): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + name: `album-${id}`, + type: "album", + updationTime: 1, + isShared: false, +}); + +const file = (id: number, collectionID: number): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: 1, + modificationTime: 1, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 1, +}); + +// A metadata-only client serving one album with one file, once. +class MockClient { + served = false; + whoami(): { email: string; userID: number } { + return { email: "u@example.com", userID: USER_ID }; + } + async collectionsSince(): Promise { + if (this.served) return { collections: [], deleted: [], cursor: 1 }; + this.served = true; + return { collections: [collection(1)], deleted: [], cursor: 1 }; + } + async filesSince(): Promise { + return { files: [file(1, 1)], deleted: [], cursor: 1 }; + } +} + +// A content source that writes a marker file and counts thumbnail fetches. +const stubSource = (): ContentSource & { thumbCalls: () => number } => { + let thumbCalls = 0; + return { + thumbCalls: () => thumbCalls, + original: async ({ destination }) => { + writeFileSync(destination, "orig-bytes"); + return { bytesWritten: 10 }; + }, + thumbnail: async ({ destination }) => { + thumbCalls++; + writeFileSync(destination, "thumb"); + return { bytesWritten: 5 }; + }, + }; +}; + +let root: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-content-lib-")); +}); + +afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); +}); + +describe("Library content wiring", () => { + it("fetches a thumbnail through a Photo and records its cache path", async () => { + const source = stubSource(); + const lib = await Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + contentSource: source, + refreshIntervalSeconds: 3600, + }); + + const photo = lib.photos.byID({ fileID: 1 }); + expect(photo).toBeDefined(); + const result = await photo!.thumbnail(); + expect(source.thumbCalls()).toBe(1); + expect(result.path).toBe(join(root, "cache", "thumbnails", "1.jpg")); + expect(existsSync(result.path)).toBe(true); + + // The cached path is now on the projected record. + expect(lib.photos.byID({ fileID: 1 })!.record().thumbnailPath).toBe( + result.path, + ); + lib.close(); + }); + + it("drives thumbnails.ensure through the cache", async () => { + const source = stubSource(); + const lib = await Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + contentSource: source, + refreshIntervalSeconds: 3600, + }); + + const results = await lib.thumbnails.ensure({ + fileIDs: [1], + priority: "visible", + }); + expect(results).toEqual([ + { fileID: 1, path: join(root, "cache", "thumbnails", "1.jpg") }, + ]); + lib.close(); + }); + + it("throws from content methods when opened without a content source", async () => { + const lib = await Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + refreshIntervalSeconds: 3600, + }); + + await expect( + lib.photos.byID({ fileID: 1 })!.thumbnail(), + ).rejects.toThrow(/content cache/i); + await expect( + lib.thumbnails.ensure({ fileIDs: [1], priority: "visible" }), + ).rejects.toThrow(/content cache/i); + lib.close(); + }); +}); diff --git a/test/library/content.test.ts b/test/library/content.test.ts new file mode 100644 index 0000000..6fe91ab --- /dev/null +++ b/test/library/content.test.ts @@ -0,0 +1,423 @@ +/** + * Tests for the on-disk content and thumbnail cache (issue #46). + * + * The cache keys stored bytes by `fileID` under `cacheDirectory`: + * `originals/.` and `thumbnails/.`. Its contract: + * + * 1. **Fetch once, then serve from disk.** The first `original`/`thumbnail` + * fetches through the request pool and stores the bytes; the next finds the + * file present and returns its path with a single `skipped` event and no + * network. A file already sitting in the backup `downloadDirectory` counts + * as present too. + * 2. **Present-means-complete.** Content appears only by the streaming atomic + * writer's rename, so a file that exists is whole. The directory listing + * taken at `open()` is the record of what is cached, and the orphan temp + * files a crashed write may have left are reaped there. + * 3. **`thumbnails.ensure` drives the thumbnail pool with priority, dedup, and + * abort.** A `fileID` asked for twice downloads once; a visible request is + * served ahead of a background one; and an `AbortSignal` drops work still + * queued while letting an in-flight fetch finish. + * + * The `ContentSource` is a stand-in: it writes deterministic bytes to the + * destination and returns the count, so the cache logic is exercised with no + * crypto and no network. Ordering tests gate the stand-in on explicit deferreds + * and assert the persisted result, never a bare call or a timer. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + mkdtempSync, + rmSync, + existsSync, + writeFileSync, + mkdirSync, + statSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + ContentCache, + type ContentSource, + type EnsureEvent, +} from "../../src/library/content.js"; +import { RequestPools } from "../../src/library/pools.js"; +import type { EnteFile } from "../../src/model/types.js"; + +const file = (id: number, title = `file-${id}.jpg`): EnteFile => ({ + id, + collectionID: 1, + ownerID: 1, + key: new Uint8Array([id & 0xff]), + metadata: { + title, + fileType: "image", + creationTime: 0, + modificationTime: 0, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 0, +}); + +// A deferred with externally callable resolve, used to gate the stand-in source +// so ordering is controlled by the test rather than by timing. +const deferred = (): { promise: Promise; resolve: () => void } => { + let resolve!: () => void; + const promise = new Promise((r) => { + resolve = r; + }); + return { promise, resolve }; +}; + +// A ContentSource that writes `${kind}:${fileID}` bytes to the destination and +// records every call. `gate` optionally blocks a call until released, and +// `completed` records the order in which fetches finished — the observable used +// by the priority and abort tests instead of a timer. +class StubSource implements ContentSource { + originalCalls: number[] = []; + thumbnailCalls: number[] = []; + completed: number[] = []; + emptyFor = new Set(); + gates = new Map>(); + + private async run( + kind: "original" | "thumbnail", + file: EnteFile, + destination: string, + ): Promise<{ bytesWritten: number }> { + const gate = this.gates.get(file.id); + if (gate) await gate; + const bytes = this.emptyFor.has(file.id) + ? new Uint8Array(0) + : new TextEncoder().encode(`${kind}:${file.id}`); + writeFileSync(destination, bytes); + this.completed.push(file.id); + return { bytesWritten: bytes.length }; + } + + async original(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + this.originalCalls.push(args.file.id); + return this.run("original", args.file, args.destination); + } + + async thumbnail(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + this.thumbnailCalls.push(args.file.id); + return this.run("thumbnail", args.file, args.destination); + } +} + +let root: string; +let cacheDir: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-content-")); + cacheDir = join(root, "cache"); +}); + +afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); +}); + +const buildCache = ( + args: { + source?: ContentSource; + files?: EnteFile[]; + pools?: RequestPools; + downloadDirectory?: string; + } = {}, +): { cache: ContentCache; source: StubSource } => { + const source = (args.source as StubSource) ?? new StubSource(); + const byID = new Map(); + for (const f of args.files ?? [file(1), file(2), file(3)]) + byID.set(f.id, f); + const cache = new ContentCache({ + pools: args.pools ?? new RequestPools(), + source, + cacheDirectory: cacheDir, + downloadDirectory: args.downloadDirectory, + getFile: (id) => byID.get(id), + }); + return { cache, source }; +}; + +describe("ContentCache.open", () => { + it("creates the cache directories with 0700 permissions", async () => { + const { cache } = buildCache(); + await cache.open(); + + const originals = join(cacheDir, "originals"); + const thumbnails = join(cacheDir, "thumbnails"); + expect(existsSync(originals)).toBe(true); + expect(existsSync(thumbnails)).toBe(true); + expect(statSync(originals).mode & 0o777).toBe(0o700); + expect(statSync(thumbnails).mode & 0o777).toBe(0o700); + }); + + it("reaps orphan temp files but keeps complete content", async () => { + const originals = join(cacheDir, "originals"); + const thumbnails = join(cacheDir, "thumbnails"); + mkdirSync(originals, { recursive: true }); + mkdirSync(thumbnails, { recursive: true }); + const orphan = join(originals, ".quak-abc123.tmp"); + const complete = join(originals, "1.jpg"); + const thumb = join(thumbnails, "2.jpg"); + writeFileSync(orphan, "half-written"); + writeFileSync(complete, "whole"); + writeFileSync(thumb, "whole-thumb"); + + const { cache } = buildCache(); + await cache.open(); + + expect(existsSync(orphan)).toBe(false); + expect(existsSync(complete)).toBe(true); + expect(existsSync(thumb)).toBe(true); + }); + + it("records already-cached files so their paths appear in pathsFor", async () => { + const originals = join(cacheDir, "originals"); + const thumbnails = join(cacheDir, "thumbnails"); + mkdirSync(originals, { recursive: true }); + mkdirSync(thumbnails, { recursive: true }); + writeFileSync(join(originals, "1.jpg"), "orig"); + writeFileSync(join(thumbnails, "1.jpg"), "thumb"); + + const { cache } = buildCache(); + await cache.open(); + + expect(cache.pathsFor(1)).toEqual({ + originalPath: join(originals, "1.jpg"), + thumbnailPath: join(thumbnails, "1.jpg"), + }); + expect(cache.pathsFor(2)).toEqual({}); + }); +}); + +describe("ContentCache.original / thumbnail", () => { + it("fetches once, then serves the cached file with a single skipped event", async () => { + const { cache, source } = buildCache(); + await cache.open(); + + const events: string[] = []; + const first = await cache.original(1, { + onProgress: (e) => events.push(e.status), + }); + expect(source.originalCalls).toEqual([1]); + expect(first.path).toBe(join(cacheDir, "originals", "1.jpg")); + expect(first.bytes).toBe("original:1".length); + expect(existsSync(first.path)).toBe(true); + expect(statSync(first.path).mode & 0o777).toBe(0o600); + expect(cache.pathsFor(1).originalPath).toBe(first.path); + + const skips: string[] = []; + const second = await cache.original(1, { + onProgress: (e) => skips.push(e.status), + }); + // No second download, and exactly one skipped event. + expect(source.originalCalls).toEqual([1]); + expect(second.path).toBe(first.path); + expect(skips).toEqual(["skipped"]); + }); + + it("serves a file already present in the download directory without fetching", async () => { + const downloadDirectory = join(root, "backup"); + mkdirSync(join(downloadDirectory, "originals"), { recursive: true }); + const backupPath = join(downloadDirectory, "originals", "1.jpg"); + writeFileSync(backupPath, "from-backup"); + + const { cache, source } = buildCache({ downloadDirectory }); + await cache.open(); + + const events: EnsureEvent["status"][] = []; + const result = await cache.original(1, { + onProgress: (e) => events.push(e.status), + }); + + expect(source.originalCalls).toEqual([]); + expect(result.path).toBe(backupPath); + expect(result.bytes).toBe("from-backup".length); + expect(events).toEqual(["skipped"]); + }); + + it("fetches and caches a thumbnail", async () => { + const { cache, source } = buildCache(); + await cache.open(); + + const result = await cache.thumbnail(2); + expect(source.thumbnailCalls).toEqual([2]); + expect(result.path).toBe(join(cacheDir, "thumbnails", "2.jpg")); + expect(existsSync(result.path)).toBe(true); + expect(cache.pathsFor(2).thumbnailPath).toBe(result.path); + }); + + it("shares one download between concurrent callers for the same file", async () => { + const { cache, source } = buildCache(); + await cache.open(); + const gate = deferred(); + source.gates.set(1, gate.promise); + + const a = cache.original(1); + const b = cache.original(1); + gate.resolve(); + const [ra, rb] = await Promise.all([a, b]); + + expect(source.originalCalls).toEqual([1]); + expect(ra.path).toBe(rb.path); + }); + + it("does not record a path when the fetched file is empty", async () => { + const { cache, source } = buildCache(); + source.emptyFor.add(1); + await cache.open(); + + await expect(cache.original(1)).rejects.toThrow(/empty/i); + expect(cache.pathsFor(1).originalPath).toBeUndefined(); + }); + + it("rejects an unknown file", async () => { + const { cache } = buildCache({ files: [] }); + await cache.open(); + await expect(cache.original(999)).rejects.toThrow(/unknown file/i); + }); +}); + +describe("ContentCache.ensureThumbnails", () => { + it("downloads once for a file listed twice and reports every id", async () => { + const { cache, source } = buildCache(); + await cache.open(); + + const results = await cache.ensureThumbnails({ + fileIDs: [1, 1, 2], + priority: "visible", + }); + + expect(source.thumbnailCalls.sort()).toEqual([1, 2]); + expect(results).toEqual([ + { fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") }, + { fileID: 2, path: join(cacheDir, "thumbnails", "2.jpg") }, + ]); + }); + + it("skips present files and reports a skipped event", async () => { + const thumbnails = join(cacheDir, "thumbnails"); + mkdirSync(thumbnails, { recursive: true }); + writeFileSync(join(thumbnails, "1.jpg"), "present"); + + const { cache, source } = buildCache(); + await cache.open(); + + const events: EnsureEvent[] = []; + const results = await cache.ensureThumbnails({ + fileIDs: [1, 2], + priority: "ahead", + onProgress: (e) => events.push(e), + }); + + expect(source.thumbnailCalls).toEqual([2]); + expect(results).toEqual([ + { fileID: 1, path: join(thumbnails, "1.jpg") }, + { fileID: 2, path: join(thumbnails, "2.jpg") }, + ]); + expect(events).toContainEqual({ + fileID: 1, + status: "skipped", + path: join(thumbnails, "1.jpg"), + }); + }); + + it("serves a visible request ahead of an already-queued background one", async () => { + // One thumbnail slot, so exactly one fetch runs at a time and the rest + // wait in the pool. A background fetch takes the slot; a background and + // a visible fetch queue behind it. When the slot frees, the pool must + // pick the visible (on-demand) request ahead of the background one that + // was submitted first. The completion order is the observable. + const pools = new RequestPools({ thumbnailConcurrency: 1 }); + const { cache, source } = buildCache({ pools }); + await cache.open(); + + const gateA = deferred(); + const gateB = deferred(); + const gateC = deferred(); + source.gates.set(1, gateA.promise); + source.gates.set(2, gateB.promise); + source.gates.set(3, gateC.promise); + + const bgFirst = cache.ensureThumbnails({ + fileIDs: [1], + priority: "background", + }); + // Let fetch 1 take the only slot before the others queue. + await Promise.resolve(); + const bgSecond = cache.ensureThumbnails({ + fileIDs: [2], + priority: "background", + }); + const visible = cache.ensureThumbnails({ + fileIDs: [3], + priority: "visible", + }); + + gateA.resolve(); + gateC.resolve(); + gateB.resolve(); + await Promise.all([bgFirst, bgSecond, visible]); + + // 1 ran first (it held the slot). Of the two that were queued, the + // visible id 3 was served before the background id 2. + expect(source.completed).toEqual([1, 3, 2]); + }); + + it("drops queued work on abort but keeps an in-flight fetch", async () => { + const pools = new RequestPools({ thumbnailConcurrency: 1 }); + const { cache, source } = buildCache({ pools }); + await cache.open(); + + const gate = deferred(); + source.gates.set(1, gate.promise); + const controller = new AbortController(); + + const pending = cache.ensureThumbnails({ + fileIDs: [1, 2], + priority: "ahead", + signal: controller.signal, + }); + // Fetch 1 is in flight (holds the slot); 2 is queued. + await Promise.resolve(); + controller.abort(); + gate.resolve(); + + const results = await pending; + + // The in-flight fetch finished and is kept; the queued one was dropped + // before it ran. + expect(source.thumbnailCalls).toEqual([1]); + expect(results).toEqual([ + { fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") }, + { fileID: 2, error: "aborted" }, + ]); + }); + + it("captures a per-file failure without failing the batch", async () => { + const { cache } = buildCache({ files: [file(1)] }); + await cache.open(); + + const results = await cache.ensureThumbnails({ + fileIDs: [1, 2], + priority: "background", + }); + + expect(results[0]).toEqual({ + fileID: 1, + path: join(cacheDir, "thumbnails", "1.jpg"), + }); + expect(results[1]?.fileID).toBe(2); + expect(results[1]?.error).toMatch(/unknown file/i); + }); +});