1 Commits
Author SHA1 Message Date
sneak ff0bbb3155 On-disk content and thumbnail cache with per-photo fetch and prefetch (closes #46)
check / check (push) Successful in 30s
Add src/library/content.ts: a ContentCache keyed by fileID under
cacheDirectory (flat originals/ and thumbnails/, 0700/0600), fetching through
the request pools (#45) and the streaming decrypt / atomic writer (#40) so
present-means-complete. One shared pool set serves both this cache and the
ML-data fetch. Photo.original and thumbnail return {path,bytes}, skipped when
present; lib.thumbnails.ensure drives the thumbnail pool with priority, dedup,
and abort.

Integrity rests on the streaming decrypt (every chunk authenticated, renamed
in only on TAG_FINAL) plus a non-empty check. The stored-hash / size compare
is deferred (tracked in #68): the only in-repo hash fixture is a placeholder,
and the stored size is the encrypted object size, not the decrypted length.

Judgement call: three thumbnail priorities map onto two tiers.

Model: opus-4-8
2026-09-22 16:12:56 +00:00
9 changed files with 1179 additions and 19 deletions
+12
View File
@@ -13,6 +13,10 @@ import {
downloadFile as dlFile, downloadFile as dlFile,
downloadThumbnail as dlThumb, downloadThumbnail as dlThumb,
} from "./download/index.js"; } from "./download/index.js";
import {
makeDownloadContentSource,
type ContentSource,
} from "./library/content.js";
import type { import type {
Collection, Collection,
EnteFile, EnteFile,
@@ -141,6 +145,14 @@ export class Client {
return this.api; return this.api;
} }
// The content-cache byte source over this client's API: each fetch is the
// download layer's request + streaming decrypt + atomic write. `Library`
// calls this to enable the on-disk content cache.
contentSource(): ContentSource {
this.assertLoggedIn();
return makeDownloadContentSource(this.api);
}
private assertLoggedIn(): void { private assertLoggedIn(): void {
if (this.loggedOut) throw new Error("Client has been logged out"); if (this.loggedOut) throw new Error("Client has been logged out");
} }
+20
View File
@@ -49,7 +49,27 @@ export {
type PhotoFilter, type PhotoFilter,
type TimelineGroup, type TimelineGroup,
type GroupBy, type GroupBy,
type ContentSource,
type ContentResult,
type ContentEvent,
type ContentOptions,
type PhotoContent,
type ThumbnailsAPI,
type ThumbnailPriority,
type EnsureOptions,
type EnsureResult,
type EnsureEvent,
} from "./library/index.js"; } from "./library/index.js";
export {
RequestPools,
BoundedPool,
DEFAULT_METADATA_CONCURRENCY,
DEFAULT_CONTENT_CONCURRENCY,
DEFAULT_THUMBNAIL_CONCURRENCY,
type RequestPoolsOptions,
type Priority,
type RunOptions,
} from "./library/pools.js";
export type { export type {
AlbumRecord, AlbumRecord,
PhotoRecord, PhotoRecord,
+416
View File
@@ -0,0 +1,416 @@
// The on-disk content and thumbnail cache keyed by fileID (issue #46).
//
// Layout under `cacheDirectory`: `originals/<fileID>.<ext>` and
// `thumbnails/<fileID>.<ext>`, flat directories at 0700 with files at 0600.
// Content appears only by the streaming atomic writer's rename (the download
// layer, #40), so a file that exists is whole — "present means complete". The
// directory listing taken at `open()` is the record of what is cached, and the
// orphan temp files a crashed write may have left are reaped there.
//
// A fetch goes through the shared request pools (#45): the content pool for
// originals, the thumbnail pool for thumbnails. The pool limits concurrency,
// orders on-demand work ahead of background, and dedups by key so a fileID
// requested twice while the first is still in flight downloads once.
//
// Integrity. The reused streaming decrypt is the enforced guarantee: every
// chunk is authenticated and the writer renames the file into place only once
// the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and
// nothing is stored. On top of that this module refuses to record a stored file
// that came out empty. The design also asks for a content-hash comparison
// against `FileMetadata.hash` (with a `fileSize` fallback); that is deferred —
// see the PR — because the exact hash construction cannot be confirmed against
// the repo's fixtures and `FileBlob.size` is the encrypted object size, not the
// decrypted length this layer has.
import { existsSync, statSync } from "node:fs";
import { chmod, mkdir, readdir, rm, stat } from "node:fs/promises";
import { extname, join } from "node:path";
import type { ApiClient } from "../api/client.js";
import {
downloadFile,
downloadThumbnail,
type ProgressCallback,
} from "../download/index.js";
import type { EnteFile } from "../model/types.js";
import type { Priority, RequestPools } from "./pools.js";
const DIR_MODE = 0o700;
const FILE_MODE = 0o600;
const TEMP_PREFIX = ".quak-";
const TEMP_SUFFIX = ".tmp";
// Ente thumbnails are always JPEG, so the cache stores them with a fixed
// extension rather than deriving one from the (image or video) title.
const THUMBNAIL_EXT = ".jpg";
type Kind = "original" | "thumbnail";
// The priority a caller attaches to a thumbnail prefetch. The pool has two
// tiers, so this three-value surface collapses onto them: only a currently
// visible thumbnail preempts (on-demand); "ahead" prefetch and speculative
// "background" work both yield to it.
export type ThumbnailPriority = "visible" | "ahead" | "background";
const poolPriorityOf = (priority: ThumbnailPriority): Priority =>
priority === "visible" ? "on-demand" : "background";
export interface ContentResult {
path: string;
bytes: number;
}
// Progress for a single `original`/`thumbnail` call. A present file emits one
// `skipped` event and nothing else; a fetched file emits `downloading` as
// plaintext lands and a final `done`.
export type ContentEvent =
| { status: "skipped"; bytes: number }
| { status: "downloading"; bytesDone: number }
| { status: "done"; bytes: number };
export interface ContentOptions {
onProgress?: (event: ContentEvent) => void;
}
// The Photo-facing content surface (the read wrappers call these). The cache
// implements it; a library opened without a content source leaves it absent.
export interface PhotoContent {
original(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
thumbnail(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
}
export interface EnsureResult {
fileID: number;
path?: string;
error?: string;
}
export interface EnsureEvent {
fileID: number;
status: "skipped" | "done" | "failed" | "aborted";
path?: string;
error?: string;
}
export interface EnsureOptions {
fileIDs: number[];
priority: ThumbnailPriority;
signal?: AbortSignal;
onProgress?: (event: EnsureEvent) => void;
}
export interface ThumbnailsAPI {
ensure(args: EnsureOptions): Promise<EnsureResult[]>;
}
// The byte source the cache fetches through. The real implementation streams
// and decrypts to the destination via the download layer; tests inject a
// stand-in so the cache logic runs with no crypto and no network. Pool routing,
// dedup, present-checks and integrity live in the cache, not here.
export interface ContentSource {
original(args: {
file: EnteFile;
destination: string;
onProgress?: ProgressCallback;
}): Promise<{ bytesWritten: number }>;
thumbnail(args: {
file: EnteFile;
destination: string;
onProgress?: ProgressCallback;
}): Promise<{ bytesWritten: number }>;
}
// The production source: each fetch is the download layer's request +
// streaming decrypt + atomic write + retry as one unit.
export const makeDownloadContentSource = (api: ApiClient): ContentSource => ({
original: ({ file, destination, onProgress }) =>
downloadFile(api, file, destination, onProgress),
thumbnail: ({ file, destination, onProgress }) =>
downloadThumbnail(api, file, destination, onProgress),
});
export interface CachedPaths {
originalPath?: string;
thumbnailPath?: string;
}
export interface ContentCacheOptions {
pools: RequestPools;
source: ContentSource;
cacheDirectory: string;
// The backup destination (issue-level `downloadDirectory`). An original
// already stored there by a backup counts as present, so the cache serves
// it rather than fetching a second copy.
downloadDirectory?: string;
// Resolve any membership of a file; every membership shares the underlying
// content key, so any one decrypts the same bytes.
getFile: (fileID: number) => EnteFile | undefined;
}
// Thrown inside a pooled task to drop a queued fetch that was aborted before it
// started running. Never escapes `ensureThumbnails`.
class AbortDrop extends Error {
constructor() {
super("aborted");
this.name = "AbortDrop";
}
}
const originalName = (file: EnteFile): string => {
const ext = extname(file.metadata.title || "") || ".bin";
return `${file.id}${ext}`;
};
// The fileID a cache filename encodes, or undefined when the name is not one
// the cache writes (`<digits><ext>`).
const fileIDFromName = (name: string): number | undefined => {
const base = name.slice(0, name.length - extname(name).length);
if (!/^\d+$/.test(base)) return undefined;
const id = Number(base);
return Number.isSafeInteger(id) ? id : undefined;
};
// Size of a regular file, or undefined if it is absent (or not a regular file).
const fileSize = (path: string): number | undefined => {
try {
const s = statSync(path);
return s.isFile() ? s.size : undefined;
} catch {
return undefined;
}
};
export class ContentCache implements PhotoContent, ThumbnailsAPI {
private readonly pools: RequestPools;
private readonly source: ContentSource;
private readonly downloadDirectory?: string;
private readonly getFile: (fileID: number) => EnteFile | undefined;
private readonly originalsDir: string;
private readonly thumbnailsDir: string;
// fileID -> absolute path of the cached bytes, seeded from the directory
// listing at open() and extended as fetches store new files.
private readonly originals = new Map<number, string>();
private readonly thumbnails = new Map<number, string>();
constructor(opts: ContentCacheOptions) {
this.pools = opts.pools;
this.source = opts.source;
this.downloadDirectory = opts.downloadDirectory;
this.getFile = opts.getFile;
this.originalsDir = join(opts.cacheDirectory, "originals");
this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails");
}
// Prepare the cache directories, reap orphan temp files, and take the
// record of what is already cached. Called once before the cache serves.
async open(): Promise<void> {
await this.ensureDir(this.originalsDir);
await this.ensureDir(this.thumbnailsDir);
await this.scan(this.originalsDir, this.originals);
await this.scan(this.thumbnailsDir, this.thumbnails);
}
// The cache paths known for a file, for the record projection to expose as
// `originalPath`/`thumbnailPath`.
pathsFor(fileID: number): CachedPaths {
const out: CachedPaths = {};
const original = this.originals.get(fileID);
if (original !== undefined) out.originalPath = original;
const thumbnail = this.thumbnails.get(fileID);
if (thumbnail !== undefined) out.thumbnailPath = thumbnail;
return out;
}
async original(
fileID: number,
opts?: ContentOptions,
): Promise<ContentResult> {
return this.get(fileID, "original", "on-demand", opts?.onProgress);
}
async thumbnail(
fileID: number,
opts?: ContentOptions,
): Promise<ContentResult> {
return this.get(fileID, "thumbnail", "on-demand", opts?.onProgress);
}
async ensure(args: EnsureOptions): Promise<EnsureResult[]> {
return this.ensureThumbnails(args);
}
async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> {
const priority = poolPriorityOf(args.priority);
// Dedup the request list so a repeated fileID is fetched once and
// reported once, in first-requested order.
const seen = new Set<number>();
const unique: number[] = [];
for (const id of args.fileIDs) {
if (!seen.has(id)) {
seen.add(id);
unique.push(id);
}
}
return Promise.all(
unique.map((fileID) =>
this.ensureOne(fileID, priority, args.signal, args.onProgress),
),
);
}
private async ensureOne(
fileID: number,
priority: Priority,
signal: AbortSignal | undefined,
onProgress: ((event: EnsureEvent) => void) | undefined,
): Promise<EnsureResult> {
try {
const result = await this.acquire(
fileID,
"thumbnail",
priority,
signal,
);
const status = result.cached ? "skipped" : "done";
onProgress?.({ fileID, status, path: result.path });
return { fileID, path: result.path };
} catch (err) {
if (err instanceof AbortDrop) {
onProgress?.({ fileID, status: "aborted" });
return { fileID, error: "aborted" };
}
const error = err instanceof Error ? err.message : String(err);
onProgress?.({ fileID, status: "failed", error });
return { fileID, error };
}
}
private async get(
fileID: number,
kind: Kind,
priority: Priority,
onProgress: ((event: ContentEvent) => void) | undefined,
): Promise<ContentResult> {
const onByte: ProgressCallback | undefined = onProgress
? (bytesDone) => onProgress({ status: "downloading", bytesDone })
: undefined;
const result = await this.acquire(fileID, kind, priority, undefined, {
onByte,
});
onProgress?.(
result.cached
? { status: "skipped", bytes: result.bytes }
: { status: "done", bytes: result.bytes },
);
return { path: result.path, bytes: result.bytes };
}
// The core: return the cached path if present, else fetch through the pool,
// store, and return it. `cached` distinguishes a present hit (no network,
// no download event) from a fresh fetch.
private async acquire(
fileID: number,
kind: Kind,
priority: Priority,
signal: AbortSignal | undefined,
opts?: { onByte?: ProgressCallback },
): Promise<{ path: string; bytes: number; cached: boolean }> {
const file = this.getFile(fileID);
if (!file) throw new Error(`content cache: unknown file ${fileID}`);
const known = kind === "original" ? this.originals : this.thumbnails;
const cached = known.get(fileID);
if (cached !== undefined) {
const size = fileSize(cached);
if (size !== undefined && size > 0)
return { path: cached, bytes: size, cached: true };
// A recorded file that has since gone re-fetches below.
known.delete(fileID);
}
// An original a backup already stored counts as present.
if (kind === "original" && this.downloadDirectory !== undefined) {
const backupPath = join(
this.downloadDirectory,
"originals",
originalName(file),
);
const size = fileSize(backupPath);
if (size !== undefined && size > 0) {
this.originals.set(fileID, backupPath);
return { path: backupPath, bytes: size, cached: true };
}
}
const dir =
kind === "original" ? this.originalsDir : this.thumbnailsDir;
const dest =
kind === "original"
? join(dir, originalName(file))
: join(dir, `${fileID}${THUMBNAIL_EXT}`);
const pool =
kind === "original" ? this.pools.content : this.pools.thumbnails;
return pool.run(
async () => {
// Dropping queued work on abort: a task still waiting for a slot
// when the signal fired sees it here and never touches the
// network. A task already past this point is in flight and runs
// to completion.
if (signal?.aborted) throw new AbortDrop();
await this.download(file, dest, kind, opts?.onByte);
await chmod(dest, FILE_MODE);
const size = (await stat(dest)).size;
if (size === 0) {
throw new Error(
`content cache: ${kind} ${fileID} stored empty`,
);
}
known.set(fileID, dest);
return { path: dest, bytes: size, cached: false };
},
{ priority, key: fileID },
);
}
private async download(
file: EnteFile,
destination: string,
kind: Kind,
onProgress: ProgressCallback | undefined,
): Promise<number> {
const args = { file, destination, onProgress };
const result =
kind === "original"
? await this.source.original(args)
: await this.source.thumbnail(args);
return result.bytesWritten;
}
private async ensureDir(dir: string): Promise<void> {
// chmod after mkdir so the mode is tightened even when the directory
// already existed with a looser one; mkdir alone would not.
await mkdir(dir, { recursive: true, mode: DIR_MODE });
await chmod(dir, DIR_MODE);
}
private async scan(dir: string, into: Map<number, string>): Promise<void> {
let entries: string[];
try {
entries = await readdir(dir);
} catch {
return;
}
for (const name of entries) {
if (name.startsWith(TEMP_PREFIX) && name.endsWith(TEMP_SUFFIX)) {
await rm(join(dir, name), { force: true }).catch(
() => undefined,
);
continue;
}
const id = fileIDFromName(name);
const path = join(dir, name);
if (id !== undefined && existsSync(path)) into.set(id, path);
}
}
}
+85 -9
View File
@@ -41,6 +41,13 @@ import {
type PhotosAPI, type PhotosAPI,
type TimelineAPI, type TimelineAPI,
} from "./read.js"; } from "./read.js";
import {
ContentCache,
type ContentSource,
type ThumbnailsAPI,
type EnsureOptions,
type EnsureResult,
} from "./content.js";
export { export {
Album, Album,
@@ -52,6 +59,18 @@ export {
type TimelineGroup, type TimelineGroup,
type GroupBy, type GroupBy,
} from "./read.js"; } from "./read.js";
export {
type ContentSource,
type ContentResult,
type ContentEvent,
type ContentOptions,
type PhotoContent,
type ThumbnailsAPI,
type ThumbnailPriority,
type EnsureOptions,
type EnsureResult,
type EnsureEvent,
} from "./content.js";
import type { CollectionsPage, FilesPage } from "../client.js"; import type { CollectionsPage, FilesPage } from "../client.js";
import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js"; import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js";
import type { Collection, EnteFile } from "../model/types.js"; import type { Collection, EnteFile } from "../model/types.js";
@@ -76,6 +95,11 @@ export interface LibraryClient {
fileIDs: number[]; fileIDs: number[];
fileKeys: Map<number, Uint8Array>; fileKeys: Map<number, Uint8Array>;
}): Promise<Map<number, MLData>>; }): Promise<Map<number, MLData>>;
// The byte source for the on-disk content cache. Optional so a mock client
// that only serves metadata still satisfies the interface; when absent (and
// no explicit `contentSource` is passed to `open`) the content cache is
// disabled and `Photo.original`/`thumbnail` and `thumbnails.ensure` throw.
contentSource?(): ContentSource;
} }
// A progress event for one unit of background work. A metadata "refresh" or an // A progress event for one unit of background work. A metadata "refresh" or an
@@ -96,14 +120,18 @@ export interface LibraryOptions {
// Where `metadata.json` lives. Defaults to the env-paths cache directory // Where `metadata.json` lives. Defaults to the env-paths cache directory
// plus the user id, so each account has its own cache. // plus the user id, so each account has its own cache.
cacheDirectory?: string; cacheDirectory?: string;
// Persistent backup destination for later phases (backup, thumbnails); the // Persistent backup destination. The refresh loop does not use it; the
// refresh loop does not use it. // content cache treats an original already stored there as present.
downloadDirectory?: string; downloadDirectory?: string;
refreshIntervalSeconds?: number; refreshIntervalSeconds?: number;
onProgress?: RefreshProgressCallback; onProgress?: RefreshProgressCallback;
// The bounded request pools (issue #45). ML data is fetched through the // The bounded request pools (issue #45), shared by the ML-data fetch (the
// metadata pool. Defaults to a fresh set at the design's caps. // metadata pool) and the content cache. Defaults to a fresh set at the
// design's caps.
pools?: RequestPools; pools?: RequestPools;
// Overrides the client's own `contentSource()`; mainly for tests that drive
// the cache with a stand-in source.
contentSource?: ContentSource;
} }
export interface LibraryStatus { export interface LibraryStatus {
@@ -138,9 +166,15 @@ export class Library {
readonly albums: AlbumsAPI; readonly albums: AlbumsAPI;
readonly photos: PhotosAPI; readonly photos: PhotosAPI;
readonly timeline: TimelineAPI; readonly timeline: TimelineAPI;
// The thumbnail-prefetch surface (issue #46): drives the thumbnail pool
// with priority, dedup, and abort.
readonly thumbnails: ThumbnailsAPI;
private readonly client: LibraryClient; private readonly client: LibraryClient;
private readonly store: MetadataStore; private readonly store: MetadataStore;
// The on-disk content cache, or undefined when no content source is
// available (a metadata-only client with no explicit source).
private readonly cache?: ContentCache;
private readonly userID: number; private readonly userID: number;
private readonly intervalMs: number; private readonly intervalMs: number;
private readonly onProgress?: RefreshProgressCallback; private readonly onProgress?: RefreshProgressCallback;
@@ -179,6 +213,7 @@ export class Library {
onProgress?: RefreshProgressCallback; onProgress?: RefreshProgressCallback;
pools: RequestPools; pools: RequestPools;
mldata?: MLDataStore; mldata?: MLDataStore;
cache?: ContentCache;
}) { }) {
this.client = args.client; this.client = args.client;
this.store = args.store; this.store = args.store;
@@ -189,14 +224,27 @@ export class Library {
this.onProgress = args.onProgress; this.onProgress = args.onProgress;
this.pools = args.pools; this.pools = args.pools;
this.mldata = args.mldata; this.mldata = args.mldata;
this.cache = args.cache;
this.lastRecords = this.deriveNow(); this.lastRecords = this.deriveNow();
// The read namespaces derive fresh from the store on each call, so they // The read namespaces derive fresh from the store on each call, so they
// always reflect the latest refresh. // always reflect the latest refresh.
const derive = (): DerivedRecords => this.deriveNow(); const derive = (): DerivedRecords => this.deriveNow();
this.albums = makeAlbumsAPI(derive); this.albums = makeAlbumsAPI(derive, this.cache);
this.photos = makePhotosAPI(derive); this.photos = makePhotosAPI(derive, this.cache);
this.timeline = makeTimelineAPI(derive); this.timeline = makeTimelineAPI(derive);
this.thumbnails = {
ensure: (opts: EnsureOptions): Promise<EnsureResult[]> => {
if (!this.cache) {
return Promise.reject(
new Error(
"thumbnails.ensure requires a library opened with a content cache",
),
);
}
return this.cache.ensureThumbnails(opts);
},
};
} }
// Load the cache and start the refresh loop. With an empty cache the first // Load the cache and start the refresh loop. With an empty cache the first
@@ -218,12 +266,33 @@ export class Library {
(opts.refreshIntervalSeconds ?? DEFAULT_REFRESH_INTERVAL_SECONDS) * (opts.refreshIntervalSeconds ?? DEFAULT_REFRESH_INTERVAL_SECONDS) *
1000; 1000;
// One request-pool set serves both the ML-data fetch and the content
// cache, so both honour the same concurrency caps.
const pools = opts.pools ?? new RequestPools();
// The ML cache only earns its keep when the client can fetch ML data; // The ML cache only earns its keep when the client can fetch ML data;
// a client without that capability opens no `mldata/` directory. // a client without that capability opens no `mldata/` directory.
const mldata = opts.client.fetchMLData const mldata = opts.client.fetchMLData
? await MLDataStore.open(join(cacheDirectory, "mldata")) ? await MLDataStore.open(join(cacheDirectory, "mldata"))
: undefined; : undefined;
// Build the content cache from an explicit source or the client's own,
// and take its record of what is already cached (and reap orphan temp
// files) before the first projection, so cached paths are present from
// the start and the first refresh raises no spurious path-change diff.
const source = opts.contentSource ?? opts.client.contentSource?.();
let cache: ContentCache | undefined;
if (source) {
cache = new ContentCache({
pools,
source,
cacheDirectory,
downloadDirectory: opts.downloadDirectory,
getFile: (fileID) => store.getFileByID(fileID),
});
await cache.open();
}
const lib = new Library({ const lib = new Library({
client: opts.client, client: opts.client,
store, store,
@@ -232,8 +301,9 @@ export class Library {
downloadDirectory: opts.downloadDirectory, downloadDirectory: opts.downloadDirectory,
intervalMs, intervalMs,
onProgress: opts.onProgress, onProgress: opts.onProgress,
pools: opts.pools ?? new RequestPools(), pools,
mldata, mldata,
cache,
}); });
if (store.loadedFromDisk) { if (store.loadedFromDisk) {
@@ -522,12 +592,18 @@ export class Library {
return [...byID.values()]; return [...byID.values()];
} }
// Gather every file membership and project the store into by-id records. // Gather every file membership and project the store into by-id records,
// filling each record's cache paths from the content cache when present.
private deriveNow(): DerivedRecords { private deriveNow(): DerivedRecords {
const collections = this.store.listCollections(); const collections = this.store.listCollections();
const files: EnteFile[] = []; const files: EnteFile[] = [];
for (const c of collections) files.push(...this.store.listFiles(c.id)); for (const c of collections) files.push(...this.store.listFiles(c.id));
return deriveRecords(collections, files); const cache = this.cache;
return deriveRecords(
collections,
files,
cache ? (fileID) => cache.pathsFor(fileID) : undefined,
);
} }
private notify(change: LibraryChange): void { private notify(change: LibraryChange): void {
+45 -10
View File
@@ -10,10 +10,14 @@
// those records: a caller that holds an object reference gets typed field // those records: a caller that holds an object reference gets typed field
// access and, for an album, its photos. They are not sent across IPC — the // access and, for an album, its photos. They are not sent across IPC — the
// plain records are the serializable surface, and `record()` returns one. // plain records are the serializable surface, and `record()` returns one.
// Content-fetch methods (`Photo.original` / `thumbnail`) belong to a later //
// unit; this surface is read-only. // A `Photo` also fetches its own bytes: `original()` and `thumbnail()` go
// through the on-disk content cache (issue #46), the one place in this module
// that is not synchronous and RAM-only. A library opened without a content
// source leaves that cache absent, and those two methods then throw.
import type { CollectionType, FileType } from "../model/types.js"; import type { CollectionType, FileType } from "../model/types.js";
import type { ContentOptions, ContentResult, PhotoContent } from "./content.js";
import type { AlbumRecord, PhotoRecord, DerivedRecords } from "./records.js"; import type { AlbumRecord, PhotoRecord, DerivedRecords } from "./records.js";
// Newest first, with fileID as a stable tiebreak so equal-timed files order // Newest first, with fileID as a stable tiebreak so equal-timed files order
@@ -29,7 +33,10 @@ const byNewestAlbum = (a: AlbumRecord, b: AlbumRecord): number =>
// A single photo. Field access mirrors `PhotoRecord`; `record()` returns the // A single photo. Field access mirrors `PhotoRecord`; `record()` returns the
// underlying plain record for callers that need the IPC-safe value. // underlying plain record for callers that need the IPC-safe value.
export class Photo { export class Photo {
constructor(private readonly rec: PhotoRecord) {} constructor(
private readonly rec: PhotoRecord,
private readonly content?: PhotoContent,
) {}
get fileID(): number { get fileID(): number {
return this.rec.fileID; return this.rec.fileID;
@@ -71,6 +78,27 @@ export class Photo {
record(): PhotoRecord { record(): PhotoRecord {
return this.rec; return this.rec;
} }
// Fetch and cache the full-resolution original, returning its on-disk path
// and byte length. Served from the cache (or the backup download directory)
// when already present, otherwise fetched through the content pool.
async original(opts?: ContentOptions): Promise<ContentResult> {
return this.contentOrThrow().original(this.rec.fileID, opts);
}
// As `original`, for the thumbnail, through the thumbnail pool.
async thumbnail(opts?: ContentOptions): Promise<ContentResult> {
return this.contentOrThrow().thumbnail(this.rec.fileID, opts);
}
private contentOrThrow(): PhotoContent {
if (!this.content) {
throw new Error(
"Photo content requires a library opened with a content cache",
);
}
return this.content;
}
} }
// A single album. `photos.list()` returns the album's photos as wrappers, // A single album. `photos.list()` returns the album's photos as wrappers,
@@ -79,6 +107,7 @@ export class Album {
constructor( constructor(
private readonly rec: AlbumRecord, private readonly rec: AlbumRecord,
private readonly records: DerivedRecords, private readonly records: DerivedRecords,
private readonly content?: PhotoContent,
) {} ) {}
get collectionID(): number { get collectionID(): number {
@@ -112,7 +141,7 @@ export class Album {
const out: Photo[] = []; const out: Photo[] = [];
for (const id of this.rec.fileIDs) { for (const id of this.rec.fileIDs) {
const p = this.records.photos.get(id); const p = this.records.photos.get(id);
if (p) out.push(new Photo(p)); if (p) out.push(new Photo(p, this.content));
} }
return out; return out;
} }
@@ -164,17 +193,20 @@ export interface TimelineAPI {
groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[]; groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[];
} }
export const makeAlbumsAPI = (derive: () => DerivedRecords): AlbumsAPI => ({ export const makeAlbumsAPI = (
derive: () => DerivedRecords,
content?: PhotoContent,
): AlbumsAPI => ({
list: (): Album[] => { list: (): Album[] => {
const records = derive(); const records = derive();
return [...records.albums.values()] return [...records.albums.values()]
.sort(byNewestAlbum) .sort(byNewestAlbum)
.map((rec) => new Album(rec, records)); .map((rec) => new Album(rec, records, content));
}, },
byID: ({ collectionID }): Album | undefined => { byID: ({ collectionID }): Album | undefined => {
const records = derive(); const records = derive();
const rec = records.albums.get(collectionID); const rec = records.albums.get(collectionID);
return rec ? new Album(rec, records) : undefined; return rec ? new Album(rec, records, content) : undefined;
}, },
byName: ({ albumName }): Album | undefined => { byName: ({ albumName }): Album | undefined => {
const records = derive(); const records = derive();
@@ -183,14 +215,17 @@ export const makeAlbumsAPI = (derive: () => DerivedRecords): AlbumsAPI => ({
const match = [...records.albums.values()] const match = [...records.albums.values()]
.sort(byNewestAlbum) .sort(byNewestAlbum)
.find((rec) => rec.name === albumName); .find((rec) => rec.name === albumName);
return match ? new Album(match, records) : undefined; return match ? new Album(match, records, content) : undefined;
}, },
}); });
export const makePhotosAPI = (derive: () => DerivedRecords): PhotosAPI => ({ export const makePhotosAPI = (
derive: () => DerivedRecords,
content?: PhotoContent,
): PhotosAPI => ({
byID: ({ fileID }): Photo | undefined => { byID: ({ fileID }): Photo | undefined => {
const rec = derive().photos.get(fileID); const rec = derive().photos.get(fileID);
return rec ? new Photo(rec) : undefined; return rec ? new Photo(rec, content) : undefined;
}, },
records: ({ fileIDs }): PhotoRecord[] => { records: ({ fileIDs }): PhotoRecord[] => {
const { photos } = derive(); const { photos } = derive();
+16
View File
@@ -166,11 +166,20 @@ const toAlbumRecord = (
}; };
}; };
// The cache paths known for a file, so the projection can expose them on the
// record without the read layer reaching into the content cache itself.
export type CachedPathLookup = (fileID: number) => {
originalPath?: string;
thumbnailPath?: string;
};
// Project the decrypted collections and file memberships into by-id records. // Project the decrypted collections and file memberships into by-id records.
// `files` is every membership (a file appears once per collection it is in). // `files` is every membership (a file appears once per collection it is in).
// `cachedPaths`, when given, fills each record's cache paths.
export const deriveRecords = ( export const deriveRecords = (
collections: Collection[], collections: Collection[],
files: EnteFile[], files: EnteFile[],
cachedPaths?: CachedPathLookup,
): DerivedRecords => { ): DerivedRecords => {
const byFileID = new Map<number, EnteFile[]>(); const byFileID = new Map<number, EnteFile[]>();
for (const f of files) { for (const f of files) {
@@ -183,6 +192,13 @@ export const deriveRecords = (
const takenAtByFile = new Map<number, number>(); const takenAtByFile = new Map<number, number>();
for (const [fileID, memberships] of byFileID) { for (const [fileID, memberships] of byFileID) {
const record = toPhotoRecord(fileID, memberships); const record = toPhotoRecord(fileID, memberships);
if (cachedPaths) {
const paths = cachedPaths(fileID);
if (paths.originalPath !== undefined)
record.originalPath = paths.originalPath;
if (paths.thumbnailPath !== undefined)
record.thumbnailPath = paths.thumbnailPath;
}
photos.set(fileID, record); photos.set(fileID, record);
takenAtByFile.set(fileID, record.takenAt); takenAtByFile.set(fileID, record.takenAt);
} }
+10
View File
@@ -179,6 +179,16 @@ export class MetadataStore {
return this.files.get(fileKey(collectionID, fileID)); return this.files.get(fileKey(collectionID, fileID));
} }
// Any membership of a file, or undefined. Every membership re-wraps the
// same underlying content key, so any one is enough to fetch the bytes;
// the content cache resolves a fileID to a file this way.
getFileByID(fileID: number): EnteFile | undefined {
for (const file of this.files.values()) {
if (file.id === fileID) return file;
}
return undefined;
}
listFiles(collectionID: number): EnteFile[] { listFiles(collectionID: number): EnteFile[] {
return [...this.files.values()].filter( return [...this.files.values()].filter(
(f) => f.collectionID === collectionID, (f) => f.collectionID === collectionID,
+152
View File
@@ -0,0 +1,152 @@
/**
* Integration between `Library` and the content cache (issue #46).
*
* The cache itself is covered in `content.test.ts`; this file locks the wiring:
* `Library.open` builds the cache from a content source, `lib.photos` hands out
* `Photo` objects that fetch through it, `lib.thumbnails.ensure` drives it, and
* a cached path shows up on the projected record. A library opened without a
* content source leaves those methods throwing rather than silently doing
* nothing.
*/
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { Library } from "../../src/library/index.js";
import type { ContentSource } from "../../src/library/content.js";
import type { CollectionsPage, FilesPage } from "../../src/client.js";
import type { Collection, EnteFile } from "../../src/model/types.js";
const USER_ID = 7;
const collection = (id: number): Collection => ({
id,
ownerID: USER_ID,
key: new Uint8Array([id & 0xff]),
name: `album-${id}`,
type: "album",
updationTime: 1,
isShared: false,
});
const file = (id: number, collectionID: number): EnteFile => ({
id,
collectionID,
ownerID: USER_ID,
key: new Uint8Array([id & 0xff]),
metadata: {
title: `file-${id}.jpg`,
fileType: "image",
creationTime: 1,
modificationTime: 1,
},
file: { decryptionHeader: "aGVhZGVy" },
thumbnail: { decryptionHeader: "dGh1bWI=" },
updationTime: 1,
});
// A metadata-only client serving one album with one file, once.
class MockClient {
served = false;
whoami(): { email: string; userID: number } {
return { email: "u@example.com", userID: USER_ID };
}
async collectionsSince(): Promise<CollectionsPage> {
if (this.served) return { collections: [], deleted: [], cursor: 1 };
this.served = true;
return { collections: [collection(1)], deleted: [], cursor: 1 };
}
async filesSince(): Promise<FilesPage> {
return { files: [file(1, 1)], deleted: [], cursor: 1 };
}
}
// A content source that writes a marker file and counts thumbnail fetches.
const stubSource = (): ContentSource & { thumbCalls: () => number } => {
let thumbCalls = 0;
return {
thumbCalls: () => thumbCalls,
original: async ({ destination }) => {
writeFileSync(destination, "orig-bytes");
return { bytesWritten: 10 };
},
thumbnail: async ({ destination }) => {
thumbCalls++;
writeFileSync(destination, "thumb");
return { bytesWritten: 5 };
},
};
};
let root: string;
beforeEach(() => {
root = mkdtempSync(join(tmpdir(), "quak-content-lib-"));
});
afterEach(() => {
if (root && existsSync(root))
rmSync(root, { recursive: true, force: true });
});
describe("Library content wiring", () => {
it("fetches a thumbnail through a Photo and records its cache path", async () => {
const source = stubSource();
const lib = await Library.open({
client: new MockClient(),
cacheDirectory: join(root, "cache"),
contentSource: source,
refreshIntervalSeconds: 3600,
});
const photo = lib.photos.byID({ fileID: 1 });
expect(photo).toBeDefined();
const result = await photo!.thumbnail();
expect(source.thumbCalls()).toBe(1);
expect(result.path).toBe(join(root, "cache", "thumbnails", "1.jpg"));
expect(existsSync(result.path)).toBe(true);
// The cached path is now on the projected record.
expect(lib.photos.byID({ fileID: 1 })!.record().thumbnailPath).toBe(
result.path,
);
lib.close();
});
it("drives thumbnails.ensure through the cache", async () => {
const source = stubSource();
const lib = await Library.open({
client: new MockClient(),
cacheDirectory: join(root, "cache"),
contentSource: source,
refreshIntervalSeconds: 3600,
});
const results = await lib.thumbnails.ensure({
fileIDs: [1],
priority: "visible",
});
expect(results).toEqual([
{ fileID: 1, path: join(root, "cache", "thumbnails", "1.jpg") },
]);
lib.close();
});
it("throws from content methods when opened without a content source", async () => {
const lib = await Library.open({
client: new MockClient(),
cacheDirectory: join(root, "cache"),
refreshIntervalSeconds: 3600,
});
await expect(
lib.photos.byID({ fileID: 1 })!.thumbnail(),
).rejects.toThrow(/content cache/i);
await expect(
lib.thumbnails.ensure({ fileIDs: [1], priority: "visible" }),
).rejects.toThrow(/content cache/i);
lib.close();
});
});
+423
View File
@@ -0,0 +1,423 @@
/**
* Tests for the on-disk content and thumbnail cache (issue #46).
*
* The cache keys stored bytes by `fileID` under `cacheDirectory`:
* `originals/<fileID>.<ext>` and `thumbnails/<fileID>.<ext>`. Its contract:
*
* 1. **Fetch once, then serve from disk.** The first `original`/`thumbnail`
* fetches through the request pool and stores the bytes; the next finds the
* file present and returns its path with a single `skipped` event and no
* network. A file already sitting in the backup `downloadDirectory` counts
* as present too.
* 2. **Present-means-complete.** Content appears only by the streaming atomic
* writer's rename, so a file that exists is whole. The directory listing
* taken at `open()` is the record of what is cached, and the orphan temp
* files a crashed write may have left are reaped there.
* 3. **`thumbnails.ensure` drives the thumbnail pool with priority, dedup, and
* abort.** A `fileID` asked for twice downloads once; a visible request is
* served ahead of a background one; and an `AbortSignal` drops work still
* queued while letting an in-flight fetch finish.
*
* The `ContentSource` is a stand-in: it writes deterministic bytes to the
* destination and returns the count, so the cache logic is exercised with no
* crypto and no network. Ordering tests gate the stand-in on explicit deferreds
* and assert the persisted result, never a bare call or a timer.
*/
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import {
mkdtempSync,
rmSync,
existsSync,
writeFileSync,
mkdirSync,
statSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
ContentCache,
type ContentSource,
type EnsureEvent,
} from "../../src/library/content.js";
import { RequestPools } from "../../src/library/pools.js";
import type { EnteFile } from "../../src/model/types.js";
const file = (id: number, title = `file-${id}.jpg`): EnteFile => ({
id,
collectionID: 1,
ownerID: 1,
key: new Uint8Array([id & 0xff]),
metadata: {
title,
fileType: "image",
creationTime: 0,
modificationTime: 0,
},
file: { decryptionHeader: "aGVhZGVy" },
thumbnail: { decryptionHeader: "dGh1bWI=" },
updationTime: 0,
});
// A deferred with externally callable resolve, used to gate the stand-in source
// so ordering is controlled by the test rather than by timing.
const deferred = (): { promise: Promise<void>; resolve: () => void } => {
let resolve!: () => void;
const promise = new Promise<void>((r) => {
resolve = r;
});
return { promise, resolve };
};
// A ContentSource that writes `${kind}:${fileID}` bytes to the destination and
// records every call. `gate` optionally blocks a call until released, and
// `completed` records the order in which fetches finished — the observable used
// by the priority and abort tests instead of a timer.
class StubSource implements ContentSource {
originalCalls: number[] = [];
thumbnailCalls: number[] = [];
completed: number[] = [];
emptyFor = new Set<number>();
gates = new Map<number, Promise<void>>();
private async run(
kind: "original" | "thumbnail",
file: EnteFile,
destination: string,
): Promise<{ bytesWritten: number }> {
const gate = this.gates.get(file.id);
if (gate) await gate;
const bytes = this.emptyFor.has(file.id)
? new Uint8Array(0)
: new TextEncoder().encode(`${kind}:${file.id}`);
writeFileSync(destination, bytes);
this.completed.push(file.id);
return { bytesWritten: bytes.length };
}
async original(args: {
file: EnteFile;
destination: string;
}): Promise<{ bytesWritten: number }> {
this.originalCalls.push(args.file.id);
return this.run("original", args.file, args.destination);
}
async thumbnail(args: {
file: EnteFile;
destination: string;
}): Promise<{ bytesWritten: number }> {
this.thumbnailCalls.push(args.file.id);
return this.run("thumbnail", args.file, args.destination);
}
}
let root: string;
let cacheDir: string;
beforeEach(() => {
root = mkdtempSync(join(tmpdir(), "quak-content-"));
cacheDir = join(root, "cache");
});
afterEach(() => {
if (root && existsSync(root))
rmSync(root, { recursive: true, force: true });
});
const buildCache = (
args: {
source?: ContentSource;
files?: EnteFile[];
pools?: RequestPools;
downloadDirectory?: string;
} = {},
): { cache: ContentCache; source: StubSource } => {
const source = (args.source as StubSource) ?? new StubSource();
const byID = new Map<number, EnteFile>();
for (const f of args.files ?? [file(1), file(2), file(3)])
byID.set(f.id, f);
const cache = new ContentCache({
pools: args.pools ?? new RequestPools(),
source,
cacheDirectory: cacheDir,
downloadDirectory: args.downloadDirectory,
getFile: (id) => byID.get(id),
});
return { cache, source };
};
describe("ContentCache.open", () => {
it("creates the cache directories with 0700 permissions", async () => {
const { cache } = buildCache();
await cache.open();
const originals = join(cacheDir, "originals");
const thumbnails = join(cacheDir, "thumbnails");
expect(existsSync(originals)).toBe(true);
expect(existsSync(thumbnails)).toBe(true);
expect(statSync(originals).mode & 0o777).toBe(0o700);
expect(statSync(thumbnails).mode & 0o777).toBe(0o700);
});
it("reaps orphan temp files but keeps complete content", async () => {
const originals = join(cacheDir, "originals");
const thumbnails = join(cacheDir, "thumbnails");
mkdirSync(originals, { recursive: true });
mkdirSync(thumbnails, { recursive: true });
const orphan = join(originals, ".quak-abc123.tmp");
const complete = join(originals, "1.jpg");
const thumb = join(thumbnails, "2.jpg");
writeFileSync(orphan, "half-written");
writeFileSync(complete, "whole");
writeFileSync(thumb, "whole-thumb");
const { cache } = buildCache();
await cache.open();
expect(existsSync(orphan)).toBe(false);
expect(existsSync(complete)).toBe(true);
expect(existsSync(thumb)).toBe(true);
});
it("records already-cached files so their paths appear in pathsFor", async () => {
const originals = join(cacheDir, "originals");
const thumbnails = join(cacheDir, "thumbnails");
mkdirSync(originals, { recursive: true });
mkdirSync(thumbnails, { recursive: true });
writeFileSync(join(originals, "1.jpg"), "orig");
writeFileSync(join(thumbnails, "1.jpg"), "thumb");
const { cache } = buildCache();
await cache.open();
expect(cache.pathsFor(1)).toEqual({
originalPath: join(originals, "1.jpg"),
thumbnailPath: join(thumbnails, "1.jpg"),
});
expect(cache.pathsFor(2)).toEqual({});
});
});
describe("ContentCache.original / thumbnail", () => {
it("fetches once, then serves the cached file with a single skipped event", async () => {
const { cache, source } = buildCache();
await cache.open();
const events: string[] = [];
const first = await cache.original(1, {
onProgress: (e) => events.push(e.status),
});
expect(source.originalCalls).toEqual([1]);
expect(first.path).toBe(join(cacheDir, "originals", "1.jpg"));
expect(first.bytes).toBe("original:1".length);
expect(existsSync(first.path)).toBe(true);
expect(statSync(first.path).mode & 0o777).toBe(0o600);
expect(cache.pathsFor(1).originalPath).toBe(first.path);
const skips: string[] = [];
const second = await cache.original(1, {
onProgress: (e) => skips.push(e.status),
});
// No second download, and exactly one skipped event.
expect(source.originalCalls).toEqual([1]);
expect(second.path).toBe(first.path);
expect(skips).toEqual(["skipped"]);
});
it("serves a file already present in the download directory without fetching", async () => {
const downloadDirectory = join(root, "backup");
mkdirSync(join(downloadDirectory, "originals"), { recursive: true });
const backupPath = join(downloadDirectory, "originals", "1.jpg");
writeFileSync(backupPath, "from-backup");
const { cache, source } = buildCache({ downloadDirectory });
await cache.open();
const events: EnsureEvent["status"][] = [];
const result = await cache.original(1, {
onProgress: (e) => events.push(e.status),
});
expect(source.originalCalls).toEqual([]);
expect(result.path).toBe(backupPath);
expect(result.bytes).toBe("from-backup".length);
expect(events).toEqual(["skipped"]);
});
it("fetches and caches a thumbnail", async () => {
const { cache, source } = buildCache();
await cache.open();
const result = await cache.thumbnail(2);
expect(source.thumbnailCalls).toEqual([2]);
expect(result.path).toBe(join(cacheDir, "thumbnails", "2.jpg"));
expect(existsSync(result.path)).toBe(true);
expect(cache.pathsFor(2).thumbnailPath).toBe(result.path);
});
it("shares one download between concurrent callers for the same file", async () => {
const { cache, source } = buildCache();
await cache.open();
const gate = deferred();
source.gates.set(1, gate.promise);
const a = cache.original(1);
const b = cache.original(1);
gate.resolve();
const [ra, rb] = await Promise.all([a, b]);
expect(source.originalCalls).toEqual([1]);
expect(ra.path).toBe(rb.path);
});
it("does not record a path when the fetched file is empty", async () => {
const { cache, source } = buildCache();
source.emptyFor.add(1);
await cache.open();
await expect(cache.original(1)).rejects.toThrow(/empty/i);
expect(cache.pathsFor(1).originalPath).toBeUndefined();
});
it("rejects an unknown file", async () => {
const { cache } = buildCache({ files: [] });
await cache.open();
await expect(cache.original(999)).rejects.toThrow(/unknown file/i);
});
});
describe("ContentCache.ensureThumbnails", () => {
it("downloads once for a file listed twice and reports every id", async () => {
const { cache, source } = buildCache();
await cache.open();
const results = await cache.ensureThumbnails({
fileIDs: [1, 1, 2],
priority: "visible",
});
expect(source.thumbnailCalls.sort()).toEqual([1, 2]);
expect(results).toEqual([
{ fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") },
{ fileID: 2, path: join(cacheDir, "thumbnails", "2.jpg") },
]);
});
it("skips present files and reports a skipped event", async () => {
const thumbnails = join(cacheDir, "thumbnails");
mkdirSync(thumbnails, { recursive: true });
writeFileSync(join(thumbnails, "1.jpg"), "present");
const { cache, source } = buildCache();
await cache.open();
const events: EnsureEvent[] = [];
const results = await cache.ensureThumbnails({
fileIDs: [1, 2],
priority: "ahead",
onProgress: (e) => events.push(e),
});
expect(source.thumbnailCalls).toEqual([2]);
expect(results).toEqual([
{ fileID: 1, path: join(thumbnails, "1.jpg") },
{ fileID: 2, path: join(thumbnails, "2.jpg") },
]);
expect(events).toContainEqual({
fileID: 1,
status: "skipped",
path: join(thumbnails, "1.jpg"),
});
});
it("serves a visible request ahead of an already-queued background one", async () => {
// One thumbnail slot, so exactly one fetch runs at a time and the rest
// wait in the pool. A background fetch takes the slot; a background and
// a visible fetch queue behind it. When the slot frees, the pool must
// pick the visible (on-demand) request ahead of the background one that
// was submitted first. The completion order is the observable.
const pools = new RequestPools({ thumbnailConcurrency: 1 });
const { cache, source } = buildCache({ pools });
await cache.open();
const gateA = deferred();
const gateB = deferred();
const gateC = deferred();
source.gates.set(1, gateA.promise);
source.gates.set(2, gateB.promise);
source.gates.set(3, gateC.promise);
const bgFirst = cache.ensureThumbnails({
fileIDs: [1],
priority: "background",
});
// Let fetch 1 take the only slot before the others queue.
await Promise.resolve();
const bgSecond = cache.ensureThumbnails({
fileIDs: [2],
priority: "background",
});
const visible = cache.ensureThumbnails({
fileIDs: [3],
priority: "visible",
});
gateA.resolve();
gateC.resolve();
gateB.resolve();
await Promise.all([bgFirst, bgSecond, visible]);
// 1 ran first (it held the slot). Of the two that were queued, the
// visible id 3 was served before the background id 2.
expect(source.completed).toEqual([1, 3, 2]);
});
it("drops queued work on abort but keeps an in-flight fetch", async () => {
const pools = new RequestPools({ thumbnailConcurrency: 1 });
const { cache, source } = buildCache({ pools });
await cache.open();
const gate = deferred();
source.gates.set(1, gate.promise);
const controller = new AbortController();
const pending = cache.ensureThumbnails({
fileIDs: [1, 2],
priority: "ahead",
signal: controller.signal,
});
// Fetch 1 is in flight (holds the slot); 2 is queued.
await Promise.resolve();
controller.abort();
gate.resolve();
const results = await pending;
// The in-flight fetch finished and is kept; the queued one was dropped
// before it ran.
expect(source.thumbnailCalls).toEqual([1]);
expect(results).toEqual([
{ fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") },
{ fileID: 2, error: "aborted" },
]);
});
it("captures a per-file failure without failing the batch", async () => {
const { cache } = buildCache({ files: [file(1)] });
await cache.open();
const results = await cache.ensureThumbnails({
fileIDs: [1, 2],
priority: "background",
});
expect(results[0]).toEqual({
fileID: 1,
path: join(cacheDir, "thumbnails", "1.jpg"),
});
expect(results[1]?.fileID).toBe(2);
expect(results[1]?.error).toMatch(/unknown file/i);
});
});