Compare commits
1
Commits
next
...
ff0bbb3155
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ff0bbb3155 |
@@ -13,6 +13,10 @@ import {
|
|||||||
downloadFile as dlFile,
|
downloadFile as dlFile,
|
||||||
downloadThumbnail as dlThumb,
|
downloadThumbnail as dlThumb,
|
||||||
} from "./download/index.js";
|
} from "./download/index.js";
|
||||||
|
import {
|
||||||
|
makeDownloadContentSource,
|
||||||
|
type ContentSource,
|
||||||
|
} from "./library/content.js";
|
||||||
import type {
|
import type {
|
||||||
Collection,
|
Collection,
|
||||||
EnteFile,
|
EnteFile,
|
||||||
@@ -141,6 +145,14 @@ export class Client {
|
|||||||
return this.api;
|
return this.api;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The content-cache byte source over this client's API: each fetch is the
|
||||||
|
// download layer's request + streaming decrypt + atomic write. `Library`
|
||||||
|
// calls this to enable the on-disk content cache.
|
||||||
|
contentSource(): ContentSource {
|
||||||
|
this.assertLoggedIn();
|
||||||
|
return makeDownloadContentSource(this.api);
|
||||||
|
}
|
||||||
|
|
||||||
private assertLoggedIn(): void {
|
private assertLoggedIn(): void {
|
||||||
if (this.loggedOut) throw new Error("Client has been logged out");
|
if (this.loggedOut) throw new Error("Client has been logged out");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,7 +49,27 @@ export {
|
|||||||
type PhotoFilter,
|
type PhotoFilter,
|
||||||
type TimelineGroup,
|
type TimelineGroup,
|
||||||
type GroupBy,
|
type GroupBy,
|
||||||
|
type ContentSource,
|
||||||
|
type ContentResult,
|
||||||
|
type ContentEvent,
|
||||||
|
type ContentOptions,
|
||||||
|
type PhotoContent,
|
||||||
|
type ThumbnailsAPI,
|
||||||
|
type ThumbnailPriority,
|
||||||
|
type EnsureOptions,
|
||||||
|
type EnsureResult,
|
||||||
|
type EnsureEvent,
|
||||||
} from "./library/index.js";
|
} from "./library/index.js";
|
||||||
|
export {
|
||||||
|
RequestPools,
|
||||||
|
BoundedPool,
|
||||||
|
DEFAULT_METADATA_CONCURRENCY,
|
||||||
|
DEFAULT_CONTENT_CONCURRENCY,
|
||||||
|
DEFAULT_THUMBNAIL_CONCURRENCY,
|
||||||
|
type RequestPoolsOptions,
|
||||||
|
type Priority,
|
||||||
|
type RunOptions,
|
||||||
|
} from "./library/pools.js";
|
||||||
export type {
|
export type {
|
||||||
AlbumRecord,
|
AlbumRecord,
|
||||||
PhotoRecord,
|
PhotoRecord,
|
||||||
|
|||||||
@@ -0,0 +1,416 @@
|
|||||||
|
// The on-disk content and thumbnail cache keyed by fileID (issue #46).
|
||||||
|
//
|
||||||
|
// Layout under `cacheDirectory`: `originals/<fileID>.<ext>` and
|
||||||
|
// `thumbnails/<fileID>.<ext>`, flat directories at 0700 with files at 0600.
|
||||||
|
// Content appears only by the streaming atomic writer's rename (the download
|
||||||
|
// layer, #40), so a file that exists is whole — "present means complete". The
|
||||||
|
// directory listing taken at `open()` is the record of what is cached, and the
|
||||||
|
// orphan temp files a crashed write may have left are reaped there.
|
||||||
|
//
|
||||||
|
// A fetch goes through the shared request pools (#45): the content pool for
|
||||||
|
// originals, the thumbnail pool for thumbnails. The pool limits concurrency,
|
||||||
|
// orders on-demand work ahead of background, and dedups by key so a fileID
|
||||||
|
// requested twice while the first is still in flight downloads once.
|
||||||
|
//
|
||||||
|
// Integrity. The reused streaming decrypt is the enforced guarantee: every
|
||||||
|
// chunk is authenticated and the writer renames the file into place only once
|
||||||
|
// the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and
|
||||||
|
// nothing is stored. On top of that this module refuses to record a stored file
|
||||||
|
// that came out empty. The design also asks for a content-hash comparison
|
||||||
|
// against `FileMetadata.hash` (with a `fileSize` fallback); that is deferred —
|
||||||
|
// see the PR — because the exact hash construction cannot be confirmed against
|
||||||
|
// the repo's fixtures and `FileBlob.size` is the encrypted object size, not the
|
||||||
|
// decrypted length this layer has.
|
||||||
|
|
||||||
|
import { existsSync, statSync } from "node:fs";
|
||||||
|
import { chmod, mkdir, readdir, rm, stat } from "node:fs/promises";
|
||||||
|
import { extname, join } from "node:path";
|
||||||
|
|
||||||
|
import type { ApiClient } from "../api/client.js";
|
||||||
|
import {
|
||||||
|
downloadFile,
|
||||||
|
downloadThumbnail,
|
||||||
|
type ProgressCallback,
|
||||||
|
} from "../download/index.js";
|
||||||
|
import type { EnteFile } from "../model/types.js";
|
||||||
|
import type { Priority, RequestPools } from "./pools.js";
|
||||||
|
|
||||||
|
const DIR_MODE = 0o700;
|
||||||
|
const FILE_MODE = 0o600;
|
||||||
|
const TEMP_PREFIX = ".quak-";
|
||||||
|
const TEMP_SUFFIX = ".tmp";
|
||||||
|
// Ente thumbnails are always JPEG, so the cache stores them with a fixed
|
||||||
|
// extension rather than deriving one from the (image or video) title.
|
||||||
|
const THUMBNAIL_EXT = ".jpg";
|
||||||
|
|
||||||
|
type Kind = "original" | "thumbnail";
|
||||||
|
|
||||||
|
// The priority a caller attaches to a thumbnail prefetch. The pool has two
|
||||||
|
// tiers, so this three-value surface collapses onto them: only a currently
|
||||||
|
// visible thumbnail preempts (on-demand); "ahead" prefetch and speculative
|
||||||
|
// "background" work both yield to it.
|
||||||
|
export type ThumbnailPriority = "visible" | "ahead" | "background";
|
||||||
|
|
||||||
|
const poolPriorityOf = (priority: ThumbnailPriority): Priority =>
|
||||||
|
priority === "visible" ? "on-demand" : "background";
|
||||||
|
|
||||||
|
export interface ContentResult {
|
||||||
|
path: string;
|
||||||
|
bytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress for a single `original`/`thumbnail` call. A present file emits one
|
||||||
|
// `skipped` event and nothing else; a fetched file emits `downloading` as
|
||||||
|
// plaintext lands and a final `done`.
|
||||||
|
export type ContentEvent =
|
||||||
|
| { status: "skipped"; bytes: number }
|
||||||
|
| { status: "downloading"; bytesDone: number }
|
||||||
|
| { status: "done"; bytes: number };
|
||||||
|
|
||||||
|
export interface ContentOptions {
|
||||||
|
onProgress?: (event: ContentEvent) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The Photo-facing content surface (the read wrappers call these). The cache
|
||||||
|
// implements it; a library opened without a content source leaves it absent.
|
||||||
|
export interface PhotoContent {
|
||||||
|
original(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
|
||||||
|
thumbnail(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EnsureResult {
|
||||||
|
fileID: number;
|
||||||
|
path?: string;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EnsureEvent {
|
||||||
|
fileID: number;
|
||||||
|
status: "skipped" | "done" | "failed" | "aborted";
|
||||||
|
path?: string;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EnsureOptions {
|
||||||
|
fileIDs: number[];
|
||||||
|
priority: ThumbnailPriority;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
onProgress?: (event: EnsureEvent) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ThumbnailsAPI {
|
||||||
|
ensure(args: EnsureOptions): Promise<EnsureResult[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The byte source the cache fetches through. The real implementation streams
|
||||||
|
// and decrypts to the destination via the download layer; tests inject a
|
||||||
|
// stand-in so the cache logic runs with no crypto and no network. Pool routing,
|
||||||
|
// dedup, present-checks and integrity live in the cache, not here.
|
||||||
|
export interface ContentSource {
|
||||||
|
original(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
onProgress?: ProgressCallback;
|
||||||
|
}): Promise<{ bytesWritten: number }>;
|
||||||
|
thumbnail(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
onProgress?: ProgressCallback;
|
||||||
|
}): Promise<{ bytesWritten: number }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The production source: each fetch is the download layer's request +
|
||||||
|
// streaming decrypt + atomic write + retry as one unit.
|
||||||
|
export const makeDownloadContentSource = (api: ApiClient): ContentSource => ({
|
||||||
|
original: ({ file, destination, onProgress }) =>
|
||||||
|
downloadFile(api, file, destination, onProgress),
|
||||||
|
thumbnail: ({ file, destination, onProgress }) =>
|
||||||
|
downloadThumbnail(api, file, destination, onProgress),
|
||||||
|
});
|
||||||
|
|
||||||
|
export interface CachedPaths {
|
||||||
|
originalPath?: string;
|
||||||
|
thumbnailPath?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ContentCacheOptions {
|
||||||
|
pools: RequestPools;
|
||||||
|
source: ContentSource;
|
||||||
|
cacheDirectory: string;
|
||||||
|
// The backup destination (issue-level `downloadDirectory`). An original
|
||||||
|
// already stored there by a backup counts as present, so the cache serves
|
||||||
|
// it rather than fetching a second copy.
|
||||||
|
downloadDirectory?: string;
|
||||||
|
// Resolve any membership of a file; every membership shares the underlying
|
||||||
|
// content key, so any one decrypts the same bytes.
|
||||||
|
getFile: (fileID: number) => EnteFile | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Thrown inside a pooled task to drop a queued fetch that was aborted before it
|
||||||
|
// started running. Never escapes `ensureThumbnails`.
|
||||||
|
class AbortDrop extends Error {
|
||||||
|
constructor() {
|
||||||
|
super("aborted");
|
||||||
|
this.name = "AbortDrop";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const originalName = (file: EnteFile): string => {
|
||||||
|
const ext = extname(file.metadata.title || "") || ".bin";
|
||||||
|
return `${file.id}${ext}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
// The fileID a cache filename encodes, or undefined when the name is not one
|
||||||
|
// the cache writes (`<digits><ext>`).
|
||||||
|
const fileIDFromName = (name: string): number | undefined => {
|
||||||
|
const base = name.slice(0, name.length - extname(name).length);
|
||||||
|
if (!/^\d+$/.test(base)) return undefined;
|
||||||
|
const id = Number(base);
|
||||||
|
return Number.isSafeInteger(id) ? id : undefined;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Size of a regular file, or undefined if it is absent (or not a regular file).
|
||||||
|
const fileSize = (path: string): number | undefined => {
|
||||||
|
try {
|
||||||
|
const s = statSync(path);
|
||||||
|
return s.isFile() ? s.size : undefined;
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||||
|
private readonly pools: RequestPools;
|
||||||
|
private readonly source: ContentSource;
|
||||||
|
private readonly downloadDirectory?: string;
|
||||||
|
private readonly getFile: (fileID: number) => EnteFile | undefined;
|
||||||
|
private readonly originalsDir: string;
|
||||||
|
private readonly thumbnailsDir: string;
|
||||||
|
// fileID -> absolute path of the cached bytes, seeded from the directory
|
||||||
|
// listing at open() and extended as fetches store new files.
|
||||||
|
private readonly originals = new Map<number, string>();
|
||||||
|
private readonly thumbnails = new Map<number, string>();
|
||||||
|
|
||||||
|
constructor(opts: ContentCacheOptions) {
|
||||||
|
this.pools = opts.pools;
|
||||||
|
this.source = opts.source;
|
||||||
|
this.downloadDirectory = opts.downloadDirectory;
|
||||||
|
this.getFile = opts.getFile;
|
||||||
|
this.originalsDir = join(opts.cacheDirectory, "originals");
|
||||||
|
this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare the cache directories, reap orphan temp files, and take the
|
||||||
|
// record of what is already cached. Called once before the cache serves.
|
||||||
|
async open(): Promise<void> {
|
||||||
|
await this.ensureDir(this.originalsDir);
|
||||||
|
await this.ensureDir(this.thumbnailsDir);
|
||||||
|
await this.scan(this.originalsDir, this.originals);
|
||||||
|
await this.scan(this.thumbnailsDir, this.thumbnails);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The cache paths known for a file, for the record projection to expose as
|
||||||
|
// `originalPath`/`thumbnailPath`.
|
||||||
|
pathsFor(fileID: number): CachedPaths {
|
||||||
|
const out: CachedPaths = {};
|
||||||
|
const original = this.originals.get(fileID);
|
||||||
|
if (original !== undefined) out.originalPath = original;
|
||||||
|
const thumbnail = this.thumbnails.get(fileID);
|
||||||
|
if (thumbnail !== undefined) out.thumbnailPath = thumbnail;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async original(
|
||||||
|
fileID: number,
|
||||||
|
opts?: ContentOptions,
|
||||||
|
): Promise<ContentResult> {
|
||||||
|
return this.get(fileID, "original", "on-demand", opts?.onProgress);
|
||||||
|
}
|
||||||
|
|
||||||
|
async thumbnail(
|
||||||
|
fileID: number,
|
||||||
|
opts?: ContentOptions,
|
||||||
|
): Promise<ContentResult> {
|
||||||
|
return this.get(fileID, "thumbnail", "on-demand", opts?.onProgress);
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensure(args: EnsureOptions): Promise<EnsureResult[]> {
|
||||||
|
return this.ensureThumbnails(args);
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> {
|
||||||
|
const priority = poolPriorityOf(args.priority);
|
||||||
|
// Dedup the request list so a repeated fileID is fetched once and
|
||||||
|
// reported once, in first-requested order.
|
||||||
|
const seen = new Set<number>();
|
||||||
|
const unique: number[] = [];
|
||||||
|
for (const id of args.fileIDs) {
|
||||||
|
if (!seen.has(id)) {
|
||||||
|
seen.add(id);
|
||||||
|
unique.push(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Promise.all(
|
||||||
|
unique.map((fileID) =>
|
||||||
|
this.ensureOne(fileID, priority, args.signal, args.onProgress),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async ensureOne(
|
||||||
|
fileID: number,
|
||||||
|
priority: Priority,
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
onProgress: ((event: EnsureEvent) => void) | undefined,
|
||||||
|
): Promise<EnsureResult> {
|
||||||
|
try {
|
||||||
|
const result = await this.acquire(
|
||||||
|
fileID,
|
||||||
|
"thumbnail",
|
||||||
|
priority,
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
const status = result.cached ? "skipped" : "done";
|
||||||
|
onProgress?.({ fileID, status, path: result.path });
|
||||||
|
return { fileID, path: result.path };
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof AbortDrop) {
|
||||||
|
onProgress?.({ fileID, status: "aborted" });
|
||||||
|
return { fileID, error: "aborted" };
|
||||||
|
}
|
||||||
|
const error = err instanceof Error ? err.message : String(err);
|
||||||
|
onProgress?.({ fileID, status: "failed", error });
|
||||||
|
return { fileID, error };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async get(
|
||||||
|
fileID: number,
|
||||||
|
kind: Kind,
|
||||||
|
priority: Priority,
|
||||||
|
onProgress: ((event: ContentEvent) => void) | undefined,
|
||||||
|
): Promise<ContentResult> {
|
||||||
|
const onByte: ProgressCallback | undefined = onProgress
|
||||||
|
? (bytesDone) => onProgress({ status: "downloading", bytesDone })
|
||||||
|
: undefined;
|
||||||
|
const result = await this.acquire(fileID, kind, priority, undefined, {
|
||||||
|
onByte,
|
||||||
|
});
|
||||||
|
onProgress?.(
|
||||||
|
result.cached
|
||||||
|
? { status: "skipped", bytes: result.bytes }
|
||||||
|
: { status: "done", bytes: result.bytes },
|
||||||
|
);
|
||||||
|
return { path: result.path, bytes: result.bytes };
|
||||||
|
}
|
||||||
|
|
||||||
|
// The core: return the cached path if present, else fetch through the pool,
|
||||||
|
// store, and return it. `cached` distinguishes a present hit (no network,
|
||||||
|
// no download event) from a fresh fetch.
|
||||||
|
private async acquire(
|
||||||
|
fileID: number,
|
||||||
|
kind: Kind,
|
||||||
|
priority: Priority,
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
opts?: { onByte?: ProgressCallback },
|
||||||
|
): Promise<{ path: string; bytes: number; cached: boolean }> {
|
||||||
|
const file = this.getFile(fileID);
|
||||||
|
if (!file) throw new Error(`content cache: unknown file ${fileID}`);
|
||||||
|
|
||||||
|
const known = kind === "original" ? this.originals : this.thumbnails;
|
||||||
|
const cached = known.get(fileID);
|
||||||
|
if (cached !== undefined) {
|
||||||
|
const size = fileSize(cached);
|
||||||
|
if (size !== undefined && size > 0)
|
||||||
|
return { path: cached, bytes: size, cached: true };
|
||||||
|
// A recorded file that has since gone re-fetches below.
|
||||||
|
known.delete(fileID);
|
||||||
|
}
|
||||||
|
|
||||||
|
// An original a backup already stored counts as present.
|
||||||
|
if (kind === "original" && this.downloadDirectory !== undefined) {
|
||||||
|
const backupPath = join(
|
||||||
|
this.downloadDirectory,
|
||||||
|
"originals",
|
||||||
|
originalName(file),
|
||||||
|
);
|
||||||
|
const size = fileSize(backupPath);
|
||||||
|
if (size !== undefined && size > 0) {
|
||||||
|
this.originals.set(fileID, backupPath);
|
||||||
|
return { path: backupPath, bytes: size, cached: true };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const dir =
|
||||||
|
kind === "original" ? this.originalsDir : this.thumbnailsDir;
|
||||||
|
const dest =
|
||||||
|
kind === "original"
|
||||||
|
? join(dir, originalName(file))
|
||||||
|
: join(dir, `${fileID}${THUMBNAIL_EXT}`);
|
||||||
|
const pool =
|
||||||
|
kind === "original" ? this.pools.content : this.pools.thumbnails;
|
||||||
|
|
||||||
|
return pool.run(
|
||||||
|
async () => {
|
||||||
|
// Dropping queued work on abort: a task still waiting for a slot
|
||||||
|
// when the signal fired sees it here and never touches the
|
||||||
|
// network. A task already past this point is in flight and runs
|
||||||
|
// to completion.
|
||||||
|
if (signal?.aborted) throw new AbortDrop();
|
||||||
|
|
||||||
|
await this.download(file, dest, kind, opts?.onByte);
|
||||||
|
await chmod(dest, FILE_MODE);
|
||||||
|
const size = (await stat(dest)).size;
|
||||||
|
if (size === 0) {
|
||||||
|
throw new Error(
|
||||||
|
`content cache: ${kind} ${fileID} stored empty`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
known.set(fileID, dest);
|
||||||
|
return { path: dest, bytes: size, cached: false };
|
||||||
|
},
|
||||||
|
{ priority, key: fileID },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async download(
|
||||||
|
file: EnteFile,
|
||||||
|
destination: string,
|
||||||
|
kind: Kind,
|
||||||
|
onProgress: ProgressCallback | undefined,
|
||||||
|
): Promise<number> {
|
||||||
|
const args = { file, destination, onProgress };
|
||||||
|
const result =
|
||||||
|
kind === "original"
|
||||||
|
? await this.source.original(args)
|
||||||
|
: await this.source.thumbnail(args);
|
||||||
|
return result.bytesWritten;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async ensureDir(dir: string): Promise<void> {
|
||||||
|
// chmod after mkdir so the mode is tightened even when the directory
|
||||||
|
// already existed with a looser one; mkdir alone would not.
|
||||||
|
await mkdir(dir, { recursive: true, mode: DIR_MODE });
|
||||||
|
await chmod(dir, DIR_MODE);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async scan(dir: string, into: Map<number, string>): Promise<void> {
|
||||||
|
let entries: string[];
|
||||||
|
try {
|
||||||
|
entries = await readdir(dir);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const name of entries) {
|
||||||
|
if (name.startsWith(TEMP_PREFIX) && name.endsWith(TEMP_SUFFIX)) {
|
||||||
|
await rm(join(dir, name), { force: true }).catch(
|
||||||
|
() => undefined,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const id = fileIDFromName(name);
|
||||||
|
const path = join(dir, name);
|
||||||
|
if (id !== undefined && existsSync(path)) into.set(id, path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+85
-9
@@ -41,6 +41,13 @@ import {
|
|||||||
type PhotosAPI,
|
type PhotosAPI,
|
||||||
type TimelineAPI,
|
type TimelineAPI,
|
||||||
} from "./read.js";
|
} from "./read.js";
|
||||||
|
import {
|
||||||
|
ContentCache,
|
||||||
|
type ContentSource,
|
||||||
|
type ThumbnailsAPI,
|
||||||
|
type EnsureOptions,
|
||||||
|
type EnsureResult,
|
||||||
|
} from "./content.js";
|
||||||
|
|
||||||
export {
|
export {
|
||||||
Album,
|
Album,
|
||||||
@@ -52,6 +59,18 @@ export {
|
|||||||
type TimelineGroup,
|
type TimelineGroup,
|
||||||
type GroupBy,
|
type GroupBy,
|
||||||
} from "./read.js";
|
} from "./read.js";
|
||||||
|
export {
|
||||||
|
type ContentSource,
|
||||||
|
type ContentResult,
|
||||||
|
type ContentEvent,
|
||||||
|
type ContentOptions,
|
||||||
|
type PhotoContent,
|
||||||
|
type ThumbnailsAPI,
|
||||||
|
type ThumbnailPriority,
|
||||||
|
type EnsureOptions,
|
||||||
|
type EnsureResult,
|
||||||
|
type EnsureEvent,
|
||||||
|
} from "./content.js";
|
||||||
import type { CollectionsPage, FilesPage } from "../client.js";
|
import type { CollectionsPage, FilesPage } from "../client.js";
|
||||||
import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js";
|
import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js";
|
||||||
import type { Collection, EnteFile } from "../model/types.js";
|
import type { Collection, EnteFile } from "../model/types.js";
|
||||||
@@ -76,6 +95,11 @@ export interface LibraryClient {
|
|||||||
fileIDs: number[];
|
fileIDs: number[];
|
||||||
fileKeys: Map<number, Uint8Array>;
|
fileKeys: Map<number, Uint8Array>;
|
||||||
}): Promise<Map<number, MLData>>;
|
}): Promise<Map<number, MLData>>;
|
||||||
|
// The byte source for the on-disk content cache. Optional so a mock client
|
||||||
|
// that only serves metadata still satisfies the interface; when absent (and
|
||||||
|
// no explicit `contentSource` is passed to `open`) the content cache is
|
||||||
|
// disabled and `Photo.original`/`thumbnail` and `thumbnails.ensure` throw.
|
||||||
|
contentSource?(): ContentSource;
|
||||||
}
|
}
|
||||||
|
|
||||||
// A progress event for one unit of background work. A metadata "refresh" or an
|
// A progress event for one unit of background work. A metadata "refresh" or an
|
||||||
@@ -96,14 +120,18 @@ export interface LibraryOptions {
|
|||||||
// Where `metadata.json` lives. Defaults to the env-paths cache directory
|
// Where `metadata.json` lives. Defaults to the env-paths cache directory
|
||||||
// plus the user id, so each account has its own cache.
|
// plus the user id, so each account has its own cache.
|
||||||
cacheDirectory?: string;
|
cacheDirectory?: string;
|
||||||
// Persistent backup destination for later phases (backup, thumbnails); the
|
// Persistent backup destination. The refresh loop does not use it; the
|
||||||
// refresh loop does not use it.
|
// content cache treats an original already stored there as present.
|
||||||
downloadDirectory?: string;
|
downloadDirectory?: string;
|
||||||
refreshIntervalSeconds?: number;
|
refreshIntervalSeconds?: number;
|
||||||
onProgress?: RefreshProgressCallback;
|
onProgress?: RefreshProgressCallback;
|
||||||
// The bounded request pools (issue #45). ML data is fetched through the
|
// The bounded request pools (issue #45), shared by the ML-data fetch (the
|
||||||
// metadata pool. Defaults to a fresh set at the design's caps.
|
// metadata pool) and the content cache. Defaults to a fresh set at the
|
||||||
|
// design's caps.
|
||||||
pools?: RequestPools;
|
pools?: RequestPools;
|
||||||
|
// Overrides the client's own `contentSource()`; mainly for tests that drive
|
||||||
|
// the cache with a stand-in source.
|
||||||
|
contentSource?: ContentSource;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LibraryStatus {
|
export interface LibraryStatus {
|
||||||
@@ -138,9 +166,15 @@ export class Library {
|
|||||||
readonly albums: AlbumsAPI;
|
readonly albums: AlbumsAPI;
|
||||||
readonly photos: PhotosAPI;
|
readonly photos: PhotosAPI;
|
||||||
readonly timeline: TimelineAPI;
|
readonly timeline: TimelineAPI;
|
||||||
|
// The thumbnail-prefetch surface (issue #46): drives the thumbnail pool
|
||||||
|
// with priority, dedup, and abort.
|
||||||
|
readonly thumbnails: ThumbnailsAPI;
|
||||||
|
|
||||||
private readonly client: LibraryClient;
|
private readonly client: LibraryClient;
|
||||||
private readonly store: MetadataStore;
|
private readonly store: MetadataStore;
|
||||||
|
// The on-disk content cache, or undefined when no content source is
|
||||||
|
// available (a metadata-only client with no explicit source).
|
||||||
|
private readonly cache?: ContentCache;
|
||||||
private readonly userID: number;
|
private readonly userID: number;
|
||||||
private readonly intervalMs: number;
|
private readonly intervalMs: number;
|
||||||
private readonly onProgress?: RefreshProgressCallback;
|
private readonly onProgress?: RefreshProgressCallback;
|
||||||
@@ -179,6 +213,7 @@ export class Library {
|
|||||||
onProgress?: RefreshProgressCallback;
|
onProgress?: RefreshProgressCallback;
|
||||||
pools: RequestPools;
|
pools: RequestPools;
|
||||||
mldata?: MLDataStore;
|
mldata?: MLDataStore;
|
||||||
|
cache?: ContentCache;
|
||||||
}) {
|
}) {
|
||||||
this.client = args.client;
|
this.client = args.client;
|
||||||
this.store = args.store;
|
this.store = args.store;
|
||||||
@@ -189,14 +224,27 @@ export class Library {
|
|||||||
this.onProgress = args.onProgress;
|
this.onProgress = args.onProgress;
|
||||||
this.pools = args.pools;
|
this.pools = args.pools;
|
||||||
this.mldata = args.mldata;
|
this.mldata = args.mldata;
|
||||||
|
this.cache = args.cache;
|
||||||
this.lastRecords = this.deriveNow();
|
this.lastRecords = this.deriveNow();
|
||||||
|
|
||||||
// The read namespaces derive fresh from the store on each call, so they
|
// The read namespaces derive fresh from the store on each call, so they
|
||||||
// always reflect the latest refresh.
|
// always reflect the latest refresh.
|
||||||
const derive = (): DerivedRecords => this.deriveNow();
|
const derive = (): DerivedRecords => this.deriveNow();
|
||||||
this.albums = makeAlbumsAPI(derive);
|
this.albums = makeAlbumsAPI(derive, this.cache);
|
||||||
this.photos = makePhotosAPI(derive);
|
this.photos = makePhotosAPI(derive, this.cache);
|
||||||
this.timeline = makeTimelineAPI(derive);
|
this.timeline = makeTimelineAPI(derive);
|
||||||
|
this.thumbnails = {
|
||||||
|
ensure: (opts: EnsureOptions): Promise<EnsureResult[]> => {
|
||||||
|
if (!this.cache) {
|
||||||
|
return Promise.reject(
|
||||||
|
new Error(
|
||||||
|
"thumbnails.ensure requires a library opened with a content cache",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return this.cache.ensureThumbnails(opts);
|
||||||
|
},
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load the cache and start the refresh loop. With an empty cache the first
|
// Load the cache and start the refresh loop. With an empty cache the first
|
||||||
@@ -218,12 +266,33 @@ export class Library {
|
|||||||
(opts.refreshIntervalSeconds ?? DEFAULT_REFRESH_INTERVAL_SECONDS) *
|
(opts.refreshIntervalSeconds ?? DEFAULT_REFRESH_INTERVAL_SECONDS) *
|
||||||
1000;
|
1000;
|
||||||
|
|
||||||
|
// One request-pool set serves both the ML-data fetch and the content
|
||||||
|
// cache, so both honour the same concurrency caps.
|
||||||
|
const pools = opts.pools ?? new RequestPools();
|
||||||
|
|
||||||
// The ML cache only earns its keep when the client can fetch ML data;
|
// The ML cache only earns its keep when the client can fetch ML data;
|
||||||
// a client without that capability opens no `mldata/` directory.
|
// a client without that capability opens no `mldata/` directory.
|
||||||
const mldata = opts.client.fetchMLData
|
const mldata = opts.client.fetchMLData
|
||||||
? await MLDataStore.open(join(cacheDirectory, "mldata"))
|
? await MLDataStore.open(join(cacheDirectory, "mldata"))
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
|
// Build the content cache from an explicit source or the client's own,
|
||||||
|
// and take its record of what is already cached (and reap orphan temp
|
||||||
|
// files) before the first projection, so cached paths are present from
|
||||||
|
// the start and the first refresh raises no spurious path-change diff.
|
||||||
|
const source = opts.contentSource ?? opts.client.contentSource?.();
|
||||||
|
let cache: ContentCache | undefined;
|
||||||
|
if (source) {
|
||||||
|
cache = new ContentCache({
|
||||||
|
pools,
|
||||||
|
source,
|
||||||
|
cacheDirectory,
|
||||||
|
downloadDirectory: opts.downloadDirectory,
|
||||||
|
getFile: (fileID) => store.getFileByID(fileID),
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
}
|
||||||
|
|
||||||
const lib = new Library({
|
const lib = new Library({
|
||||||
client: opts.client,
|
client: opts.client,
|
||||||
store,
|
store,
|
||||||
@@ -232,8 +301,9 @@ export class Library {
|
|||||||
downloadDirectory: opts.downloadDirectory,
|
downloadDirectory: opts.downloadDirectory,
|
||||||
intervalMs,
|
intervalMs,
|
||||||
onProgress: opts.onProgress,
|
onProgress: opts.onProgress,
|
||||||
pools: opts.pools ?? new RequestPools(),
|
pools,
|
||||||
mldata,
|
mldata,
|
||||||
|
cache,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (store.loadedFromDisk) {
|
if (store.loadedFromDisk) {
|
||||||
@@ -522,12 +592,18 @@ export class Library {
|
|||||||
return [...byID.values()];
|
return [...byID.values()];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Gather every file membership and project the store into by-id records.
|
// Gather every file membership and project the store into by-id records,
|
||||||
|
// filling each record's cache paths from the content cache when present.
|
||||||
private deriveNow(): DerivedRecords {
|
private deriveNow(): DerivedRecords {
|
||||||
const collections = this.store.listCollections();
|
const collections = this.store.listCollections();
|
||||||
const files: EnteFile[] = [];
|
const files: EnteFile[] = [];
|
||||||
for (const c of collections) files.push(...this.store.listFiles(c.id));
|
for (const c of collections) files.push(...this.store.listFiles(c.id));
|
||||||
return deriveRecords(collections, files);
|
const cache = this.cache;
|
||||||
|
return deriveRecords(
|
||||||
|
collections,
|
||||||
|
files,
|
||||||
|
cache ? (fileID) => cache.pathsFor(fileID) : undefined,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private notify(change: LibraryChange): void {
|
private notify(change: LibraryChange): void {
|
||||||
|
|||||||
+45
-10
@@ -10,10 +10,14 @@
|
|||||||
// those records: a caller that holds an object reference gets typed field
|
// those records: a caller that holds an object reference gets typed field
|
||||||
// access and, for an album, its photos. They are not sent across IPC — the
|
// access and, for an album, its photos. They are not sent across IPC — the
|
||||||
// plain records are the serializable surface, and `record()` returns one.
|
// plain records are the serializable surface, and `record()` returns one.
|
||||||
// Content-fetch methods (`Photo.original` / `thumbnail`) belong to a later
|
//
|
||||||
// unit; this surface is read-only.
|
// A `Photo` also fetches its own bytes: `original()` and `thumbnail()` go
|
||||||
|
// through the on-disk content cache (issue #46), the one place in this module
|
||||||
|
// that is not synchronous and RAM-only. A library opened without a content
|
||||||
|
// source leaves that cache absent, and those two methods then throw.
|
||||||
|
|
||||||
import type { CollectionType, FileType } from "../model/types.js";
|
import type { CollectionType, FileType } from "../model/types.js";
|
||||||
|
import type { ContentOptions, ContentResult, PhotoContent } from "./content.js";
|
||||||
import type { AlbumRecord, PhotoRecord, DerivedRecords } from "./records.js";
|
import type { AlbumRecord, PhotoRecord, DerivedRecords } from "./records.js";
|
||||||
|
|
||||||
// Newest first, with fileID as a stable tiebreak so equal-timed files order
|
// Newest first, with fileID as a stable tiebreak so equal-timed files order
|
||||||
@@ -29,7 +33,10 @@ const byNewestAlbum = (a: AlbumRecord, b: AlbumRecord): number =>
|
|||||||
// A single photo. Field access mirrors `PhotoRecord`; `record()` returns the
|
// A single photo. Field access mirrors `PhotoRecord`; `record()` returns the
|
||||||
// underlying plain record for callers that need the IPC-safe value.
|
// underlying plain record for callers that need the IPC-safe value.
|
||||||
export class Photo {
|
export class Photo {
|
||||||
constructor(private readonly rec: PhotoRecord) {}
|
constructor(
|
||||||
|
private readonly rec: PhotoRecord,
|
||||||
|
private readonly content?: PhotoContent,
|
||||||
|
) {}
|
||||||
|
|
||||||
get fileID(): number {
|
get fileID(): number {
|
||||||
return this.rec.fileID;
|
return this.rec.fileID;
|
||||||
@@ -71,6 +78,27 @@ export class Photo {
|
|||||||
record(): PhotoRecord {
|
record(): PhotoRecord {
|
||||||
return this.rec;
|
return this.rec;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fetch and cache the full-resolution original, returning its on-disk path
|
||||||
|
// and byte length. Served from the cache (or the backup download directory)
|
||||||
|
// when already present, otherwise fetched through the content pool.
|
||||||
|
async original(opts?: ContentOptions): Promise<ContentResult> {
|
||||||
|
return this.contentOrThrow().original(this.rec.fileID, opts);
|
||||||
|
}
|
||||||
|
|
||||||
|
// As `original`, for the thumbnail, through the thumbnail pool.
|
||||||
|
async thumbnail(opts?: ContentOptions): Promise<ContentResult> {
|
||||||
|
return this.contentOrThrow().thumbnail(this.rec.fileID, opts);
|
||||||
|
}
|
||||||
|
|
||||||
|
private contentOrThrow(): PhotoContent {
|
||||||
|
if (!this.content) {
|
||||||
|
throw new Error(
|
||||||
|
"Photo content requires a library opened with a content cache",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return this.content;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A single album. `photos.list()` returns the album's photos as wrappers,
|
// A single album. `photos.list()` returns the album's photos as wrappers,
|
||||||
@@ -79,6 +107,7 @@ export class Album {
|
|||||||
constructor(
|
constructor(
|
||||||
private readonly rec: AlbumRecord,
|
private readonly rec: AlbumRecord,
|
||||||
private readonly records: DerivedRecords,
|
private readonly records: DerivedRecords,
|
||||||
|
private readonly content?: PhotoContent,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
get collectionID(): number {
|
get collectionID(): number {
|
||||||
@@ -112,7 +141,7 @@ export class Album {
|
|||||||
const out: Photo[] = [];
|
const out: Photo[] = [];
|
||||||
for (const id of this.rec.fileIDs) {
|
for (const id of this.rec.fileIDs) {
|
||||||
const p = this.records.photos.get(id);
|
const p = this.records.photos.get(id);
|
||||||
if (p) out.push(new Photo(p));
|
if (p) out.push(new Photo(p, this.content));
|
||||||
}
|
}
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
@@ -164,17 +193,20 @@ export interface TimelineAPI {
|
|||||||
groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[];
|
groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export const makeAlbumsAPI = (derive: () => DerivedRecords): AlbumsAPI => ({
|
export const makeAlbumsAPI = (
|
||||||
|
derive: () => DerivedRecords,
|
||||||
|
content?: PhotoContent,
|
||||||
|
): AlbumsAPI => ({
|
||||||
list: (): Album[] => {
|
list: (): Album[] => {
|
||||||
const records = derive();
|
const records = derive();
|
||||||
return [...records.albums.values()]
|
return [...records.albums.values()]
|
||||||
.sort(byNewestAlbum)
|
.sort(byNewestAlbum)
|
||||||
.map((rec) => new Album(rec, records));
|
.map((rec) => new Album(rec, records, content));
|
||||||
},
|
},
|
||||||
byID: ({ collectionID }): Album | undefined => {
|
byID: ({ collectionID }): Album | undefined => {
|
||||||
const records = derive();
|
const records = derive();
|
||||||
const rec = records.albums.get(collectionID);
|
const rec = records.albums.get(collectionID);
|
||||||
return rec ? new Album(rec, records) : undefined;
|
return rec ? new Album(rec, records, content) : undefined;
|
||||||
},
|
},
|
||||||
byName: ({ albumName }): Album | undefined => {
|
byName: ({ albumName }): Album | undefined => {
|
||||||
const records = derive();
|
const records = derive();
|
||||||
@@ -183,14 +215,17 @@ export const makeAlbumsAPI = (derive: () => DerivedRecords): AlbumsAPI => ({
|
|||||||
const match = [...records.albums.values()]
|
const match = [...records.albums.values()]
|
||||||
.sort(byNewestAlbum)
|
.sort(byNewestAlbum)
|
||||||
.find((rec) => rec.name === albumName);
|
.find((rec) => rec.name === albumName);
|
||||||
return match ? new Album(match, records) : undefined;
|
return match ? new Album(match, records, content) : undefined;
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
export const makePhotosAPI = (derive: () => DerivedRecords): PhotosAPI => ({
|
export const makePhotosAPI = (
|
||||||
|
derive: () => DerivedRecords,
|
||||||
|
content?: PhotoContent,
|
||||||
|
): PhotosAPI => ({
|
||||||
byID: ({ fileID }): Photo | undefined => {
|
byID: ({ fileID }): Photo | undefined => {
|
||||||
const rec = derive().photos.get(fileID);
|
const rec = derive().photos.get(fileID);
|
||||||
return rec ? new Photo(rec) : undefined;
|
return rec ? new Photo(rec, content) : undefined;
|
||||||
},
|
},
|
||||||
records: ({ fileIDs }): PhotoRecord[] => {
|
records: ({ fileIDs }): PhotoRecord[] => {
|
||||||
const { photos } = derive();
|
const { photos } = derive();
|
||||||
|
|||||||
@@ -166,11 +166,20 @@ const toAlbumRecord = (
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// The cache paths known for a file, so the projection can expose them on the
|
||||||
|
// record without the read layer reaching into the content cache itself.
|
||||||
|
export type CachedPathLookup = (fileID: number) => {
|
||||||
|
originalPath?: string;
|
||||||
|
thumbnailPath?: string;
|
||||||
|
};
|
||||||
|
|
||||||
// Project the decrypted collections and file memberships into by-id records.
|
// Project the decrypted collections and file memberships into by-id records.
|
||||||
// `files` is every membership (a file appears once per collection it is in).
|
// `files` is every membership (a file appears once per collection it is in).
|
||||||
|
// `cachedPaths`, when given, fills each record's cache paths.
|
||||||
export const deriveRecords = (
|
export const deriveRecords = (
|
||||||
collections: Collection[],
|
collections: Collection[],
|
||||||
files: EnteFile[],
|
files: EnteFile[],
|
||||||
|
cachedPaths?: CachedPathLookup,
|
||||||
): DerivedRecords => {
|
): DerivedRecords => {
|
||||||
const byFileID = new Map<number, EnteFile[]>();
|
const byFileID = new Map<number, EnteFile[]>();
|
||||||
for (const f of files) {
|
for (const f of files) {
|
||||||
@@ -183,6 +192,13 @@ export const deriveRecords = (
|
|||||||
const takenAtByFile = new Map<number, number>();
|
const takenAtByFile = new Map<number, number>();
|
||||||
for (const [fileID, memberships] of byFileID) {
|
for (const [fileID, memberships] of byFileID) {
|
||||||
const record = toPhotoRecord(fileID, memberships);
|
const record = toPhotoRecord(fileID, memberships);
|
||||||
|
if (cachedPaths) {
|
||||||
|
const paths = cachedPaths(fileID);
|
||||||
|
if (paths.originalPath !== undefined)
|
||||||
|
record.originalPath = paths.originalPath;
|
||||||
|
if (paths.thumbnailPath !== undefined)
|
||||||
|
record.thumbnailPath = paths.thumbnailPath;
|
||||||
|
}
|
||||||
photos.set(fileID, record);
|
photos.set(fileID, record);
|
||||||
takenAtByFile.set(fileID, record.takenAt);
|
takenAtByFile.set(fileID, record.takenAt);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -179,6 +179,16 @@ export class MetadataStore {
|
|||||||
return this.files.get(fileKey(collectionID, fileID));
|
return this.files.get(fileKey(collectionID, fileID));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Any membership of a file, or undefined. Every membership re-wraps the
|
||||||
|
// same underlying content key, so any one is enough to fetch the bytes;
|
||||||
|
// the content cache resolves a fileID to a file this way.
|
||||||
|
getFileByID(fileID: number): EnteFile | undefined {
|
||||||
|
for (const file of this.files.values()) {
|
||||||
|
if (file.id === fileID) return file;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
listFiles(collectionID: number): EnteFile[] {
|
listFiles(collectionID: number): EnteFile[] {
|
||||||
return [...this.files.values()].filter(
|
return [...this.files.values()].filter(
|
||||||
(f) => f.collectionID === collectionID,
|
(f) => f.collectionID === collectionID,
|
||||||
|
|||||||
@@ -0,0 +1,152 @@
|
|||||||
|
/**
|
||||||
|
* Integration between `Library` and the content cache (issue #46).
|
||||||
|
*
|
||||||
|
* The cache itself is covered in `content.test.ts`; this file locks the wiring:
|
||||||
|
* `Library.open` builds the cache from a content source, `lib.photos` hands out
|
||||||
|
* `Photo` objects that fetch through it, `lib.thumbnails.ensure` drives it, and
|
||||||
|
* a cached path shows up on the projected record. A library opened without a
|
||||||
|
* content source leaves those methods throwing rather than silently doing
|
||||||
|
* nothing.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { Library } from "../../src/library/index.js";
|
||||||
|
import type { ContentSource } from "../../src/library/content.js";
|
||||||
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
||||||
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||||
|
|
||||||
|
const USER_ID = 7;
|
||||||
|
|
||||||
|
const collection = (id: number): Collection => ({
|
||||||
|
id,
|
||||||
|
ownerID: USER_ID,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
name: `album-${id}`,
|
||||||
|
type: "album",
|
||||||
|
updationTime: 1,
|
||||||
|
isShared: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const file = (id: number, collectionID: number): EnteFile => ({
|
||||||
|
id,
|
||||||
|
collectionID,
|
||||||
|
ownerID: USER_ID,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
metadata: {
|
||||||
|
title: `file-${id}.jpg`,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: 1,
|
||||||
|
modificationTime: 1,
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: "aGVhZGVy" },
|
||||||
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
||||||
|
updationTime: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
// A metadata-only client serving one album with one file, once.
|
||||||
|
class MockClient {
|
||||||
|
served = false;
|
||||||
|
whoami(): { email: string; userID: number } {
|
||||||
|
return { email: "u@example.com", userID: USER_ID };
|
||||||
|
}
|
||||||
|
async collectionsSince(): Promise<CollectionsPage> {
|
||||||
|
if (this.served) return { collections: [], deleted: [], cursor: 1 };
|
||||||
|
this.served = true;
|
||||||
|
return { collections: [collection(1)], deleted: [], cursor: 1 };
|
||||||
|
}
|
||||||
|
async filesSince(): Promise<FilesPage> {
|
||||||
|
return { files: [file(1, 1)], deleted: [], cursor: 1 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A content source that writes a marker file and counts thumbnail fetches.
|
||||||
|
const stubSource = (): ContentSource & { thumbCalls: () => number } => {
|
||||||
|
let thumbCalls = 0;
|
||||||
|
return {
|
||||||
|
thumbCalls: () => thumbCalls,
|
||||||
|
original: async ({ destination }) => {
|
||||||
|
writeFileSync(destination, "orig-bytes");
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ destination }) => {
|
||||||
|
thumbCalls++;
|
||||||
|
writeFileSync(destination, "thumb");
|
||||||
|
return { bytesWritten: 5 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-content-lib-"));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Library content wiring", () => {
|
||||||
|
it("fetches a thumbnail through a Photo and records its cache path", async () => {
|
||||||
|
const source = stubSource();
|
||||||
|
const lib = await Library.open({
|
||||||
|
client: new MockClient(),
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
contentSource: source,
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
});
|
||||||
|
|
||||||
|
const photo = lib.photos.byID({ fileID: 1 });
|
||||||
|
expect(photo).toBeDefined();
|
||||||
|
const result = await photo!.thumbnail();
|
||||||
|
expect(source.thumbCalls()).toBe(1);
|
||||||
|
expect(result.path).toBe(join(root, "cache", "thumbnails", "1.jpg"));
|
||||||
|
expect(existsSync(result.path)).toBe(true);
|
||||||
|
|
||||||
|
// The cached path is now on the projected record.
|
||||||
|
expect(lib.photos.byID({ fileID: 1 })!.record().thumbnailPath).toBe(
|
||||||
|
result.path,
|
||||||
|
);
|
||||||
|
lib.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drives thumbnails.ensure through the cache", async () => {
|
||||||
|
const source = stubSource();
|
||||||
|
const lib = await Library.open({
|
||||||
|
client: new MockClient(),
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
contentSource: source,
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await lib.thumbnails.ensure({
|
||||||
|
fileIDs: [1],
|
||||||
|
priority: "visible",
|
||||||
|
});
|
||||||
|
expect(results).toEqual([
|
||||||
|
{ fileID: 1, path: join(root, "cache", "thumbnails", "1.jpg") },
|
||||||
|
]);
|
||||||
|
lib.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws from content methods when opened without a content source", async () => {
|
||||||
|
const lib = await Library.open({
|
||||||
|
client: new MockClient(),
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
lib.photos.byID({ fileID: 1 })!.thumbnail(),
|
||||||
|
).rejects.toThrow(/content cache/i);
|
||||||
|
await expect(
|
||||||
|
lib.thumbnails.ensure({ fileIDs: [1], priority: "visible" }),
|
||||||
|
).rejects.toThrow(/content cache/i);
|
||||||
|
lib.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,423 @@
|
|||||||
|
/**
|
||||||
|
* Tests for the on-disk content and thumbnail cache (issue #46).
|
||||||
|
*
|
||||||
|
* The cache keys stored bytes by `fileID` under `cacheDirectory`:
|
||||||
|
* `originals/<fileID>.<ext>` and `thumbnails/<fileID>.<ext>`. Its contract:
|
||||||
|
*
|
||||||
|
* 1. **Fetch once, then serve from disk.** The first `original`/`thumbnail`
|
||||||
|
* fetches through the request pool and stores the bytes; the next finds the
|
||||||
|
* file present and returns its path with a single `skipped` event and no
|
||||||
|
* network. A file already sitting in the backup `downloadDirectory` counts
|
||||||
|
* as present too.
|
||||||
|
* 2. **Present-means-complete.** Content appears only by the streaming atomic
|
||||||
|
* writer's rename, so a file that exists is whole. The directory listing
|
||||||
|
* taken at `open()` is the record of what is cached, and the orphan temp
|
||||||
|
* files a crashed write may have left are reaped there.
|
||||||
|
* 3. **`thumbnails.ensure` drives the thumbnail pool with priority, dedup, and
|
||||||
|
* abort.** A `fileID` asked for twice downloads once; a visible request is
|
||||||
|
* served ahead of a background one; and an `AbortSignal` drops work still
|
||||||
|
* queued while letting an in-flight fetch finish.
|
||||||
|
*
|
||||||
|
* The `ContentSource` is a stand-in: it writes deterministic bytes to the
|
||||||
|
* destination and returns the count, so the cache logic is exercised with no
|
||||||
|
* crypto and no network. Ordering tests gate the stand-in on explicit deferreds
|
||||||
|
* and assert the persisted result, never a bare call or a timer.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import {
|
||||||
|
mkdtempSync,
|
||||||
|
rmSync,
|
||||||
|
existsSync,
|
||||||
|
writeFileSync,
|
||||||
|
mkdirSync,
|
||||||
|
statSync,
|
||||||
|
} from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import {
|
||||||
|
ContentCache,
|
||||||
|
type ContentSource,
|
||||||
|
type EnsureEvent,
|
||||||
|
} from "../../src/library/content.js";
|
||||||
|
import { RequestPools } from "../../src/library/pools.js";
|
||||||
|
import type { EnteFile } from "../../src/model/types.js";
|
||||||
|
|
||||||
|
const file = (id: number, title = `file-${id}.jpg`): EnteFile => ({
|
||||||
|
id,
|
||||||
|
collectionID: 1,
|
||||||
|
ownerID: 1,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
metadata: {
|
||||||
|
title,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: 0,
|
||||||
|
modificationTime: 0,
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: "aGVhZGVy" },
|
||||||
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
||||||
|
updationTime: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
// A deferred with externally callable resolve, used to gate the stand-in source
|
||||||
|
// so ordering is controlled by the test rather than by timing.
|
||||||
|
const deferred = (): { promise: Promise<void>; resolve: () => void } => {
|
||||||
|
let resolve!: () => void;
|
||||||
|
const promise = new Promise<void>((r) => {
|
||||||
|
resolve = r;
|
||||||
|
});
|
||||||
|
return { promise, resolve };
|
||||||
|
};
|
||||||
|
|
||||||
|
// A ContentSource that writes `${kind}:${fileID}` bytes to the destination and
|
||||||
|
// records every call. `gate` optionally blocks a call until released, and
|
||||||
|
// `completed` records the order in which fetches finished — the observable used
|
||||||
|
// by the priority and abort tests instead of a timer.
|
||||||
|
class StubSource implements ContentSource {
|
||||||
|
originalCalls: number[] = [];
|
||||||
|
thumbnailCalls: number[] = [];
|
||||||
|
completed: number[] = [];
|
||||||
|
emptyFor = new Set<number>();
|
||||||
|
gates = new Map<number, Promise<void>>();
|
||||||
|
|
||||||
|
private async run(
|
||||||
|
kind: "original" | "thumbnail",
|
||||||
|
file: EnteFile,
|
||||||
|
destination: string,
|
||||||
|
): Promise<{ bytesWritten: number }> {
|
||||||
|
const gate = this.gates.get(file.id);
|
||||||
|
if (gate) await gate;
|
||||||
|
const bytes = this.emptyFor.has(file.id)
|
||||||
|
? new Uint8Array(0)
|
||||||
|
: new TextEncoder().encode(`${kind}:${file.id}`);
|
||||||
|
writeFileSync(destination, bytes);
|
||||||
|
this.completed.push(file.id);
|
||||||
|
return { bytesWritten: bytes.length };
|
||||||
|
}
|
||||||
|
|
||||||
|
async original(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
}): Promise<{ bytesWritten: number }> {
|
||||||
|
this.originalCalls.push(args.file.id);
|
||||||
|
return this.run("original", args.file, args.destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
async thumbnail(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
}): Promise<{ bytesWritten: number }> {
|
||||||
|
this.thumbnailCalls.push(args.file.id);
|
||||||
|
return this.run("thumbnail", args.file, args.destination);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let root: string;
|
||||||
|
let cacheDir: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-content-"));
|
||||||
|
cacheDir = join(root, "cache");
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
const buildCache = (
|
||||||
|
args: {
|
||||||
|
source?: ContentSource;
|
||||||
|
files?: EnteFile[];
|
||||||
|
pools?: RequestPools;
|
||||||
|
downloadDirectory?: string;
|
||||||
|
} = {},
|
||||||
|
): { cache: ContentCache; source: StubSource } => {
|
||||||
|
const source = (args.source as StubSource) ?? new StubSource();
|
||||||
|
const byID = new Map<number, EnteFile>();
|
||||||
|
for (const f of args.files ?? [file(1), file(2), file(3)])
|
||||||
|
byID.set(f.id, f);
|
||||||
|
const cache = new ContentCache({
|
||||||
|
pools: args.pools ?? new RequestPools(),
|
||||||
|
source,
|
||||||
|
cacheDirectory: cacheDir,
|
||||||
|
downloadDirectory: args.downloadDirectory,
|
||||||
|
getFile: (id) => byID.get(id),
|
||||||
|
});
|
||||||
|
return { cache, source };
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("ContentCache.open", () => {
|
||||||
|
it("creates the cache directories with 0700 permissions", async () => {
|
||||||
|
const { cache } = buildCache();
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const originals = join(cacheDir, "originals");
|
||||||
|
const thumbnails = join(cacheDir, "thumbnails");
|
||||||
|
expect(existsSync(originals)).toBe(true);
|
||||||
|
expect(existsSync(thumbnails)).toBe(true);
|
||||||
|
expect(statSync(originals).mode & 0o777).toBe(0o700);
|
||||||
|
expect(statSync(thumbnails).mode & 0o777).toBe(0o700);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reaps orphan temp files but keeps complete content", async () => {
|
||||||
|
const originals = join(cacheDir, "originals");
|
||||||
|
const thumbnails = join(cacheDir, "thumbnails");
|
||||||
|
mkdirSync(originals, { recursive: true });
|
||||||
|
mkdirSync(thumbnails, { recursive: true });
|
||||||
|
const orphan = join(originals, ".quak-abc123.tmp");
|
||||||
|
const complete = join(originals, "1.jpg");
|
||||||
|
const thumb = join(thumbnails, "2.jpg");
|
||||||
|
writeFileSync(orphan, "half-written");
|
||||||
|
writeFileSync(complete, "whole");
|
||||||
|
writeFileSync(thumb, "whole-thumb");
|
||||||
|
|
||||||
|
const { cache } = buildCache();
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
expect(existsSync(orphan)).toBe(false);
|
||||||
|
expect(existsSync(complete)).toBe(true);
|
||||||
|
expect(existsSync(thumb)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("records already-cached files so their paths appear in pathsFor", async () => {
|
||||||
|
const originals = join(cacheDir, "originals");
|
||||||
|
const thumbnails = join(cacheDir, "thumbnails");
|
||||||
|
mkdirSync(originals, { recursive: true });
|
||||||
|
mkdirSync(thumbnails, { recursive: true });
|
||||||
|
writeFileSync(join(originals, "1.jpg"), "orig");
|
||||||
|
writeFileSync(join(thumbnails, "1.jpg"), "thumb");
|
||||||
|
|
||||||
|
const { cache } = buildCache();
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
expect(cache.pathsFor(1)).toEqual({
|
||||||
|
originalPath: join(originals, "1.jpg"),
|
||||||
|
thumbnailPath: join(thumbnails, "1.jpg"),
|
||||||
|
});
|
||||||
|
expect(cache.pathsFor(2)).toEqual({});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("ContentCache.original / thumbnail", () => {
|
||||||
|
it("fetches once, then serves the cached file with a single skipped event", async () => {
|
||||||
|
const { cache, source } = buildCache();
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const events: string[] = [];
|
||||||
|
const first = await cache.original(1, {
|
||||||
|
onProgress: (e) => events.push(e.status),
|
||||||
|
});
|
||||||
|
expect(source.originalCalls).toEqual([1]);
|
||||||
|
expect(first.path).toBe(join(cacheDir, "originals", "1.jpg"));
|
||||||
|
expect(first.bytes).toBe("original:1".length);
|
||||||
|
expect(existsSync(first.path)).toBe(true);
|
||||||
|
expect(statSync(first.path).mode & 0o777).toBe(0o600);
|
||||||
|
expect(cache.pathsFor(1).originalPath).toBe(first.path);
|
||||||
|
|
||||||
|
const skips: string[] = [];
|
||||||
|
const second = await cache.original(1, {
|
||||||
|
onProgress: (e) => skips.push(e.status),
|
||||||
|
});
|
||||||
|
// No second download, and exactly one skipped event.
|
||||||
|
expect(source.originalCalls).toEqual([1]);
|
||||||
|
expect(second.path).toBe(first.path);
|
||||||
|
expect(skips).toEqual(["skipped"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("serves a file already present in the download directory without fetching", async () => {
|
||||||
|
const downloadDirectory = join(root, "backup");
|
||||||
|
mkdirSync(join(downloadDirectory, "originals"), { recursive: true });
|
||||||
|
const backupPath = join(downloadDirectory, "originals", "1.jpg");
|
||||||
|
writeFileSync(backupPath, "from-backup");
|
||||||
|
|
||||||
|
const { cache, source } = buildCache({ downloadDirectory });
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const events: EnsureEvent["status"][] = [];
|
||||||
|
const result = await cache.original(1, {
|
||||||
|
onProgress: (e) => events.push(e.status),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(source.originalCalls).toEqual([]);
|
||||||
|
expect(result.path).toBe(backupPath);
|
||||||
|
expect(result.bytes).toBe("from-backup".length);
|
||||||
|
expect(events).toEqual(["skipped"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fetches and caches a thumbnail", async () => {
|
||||||
|
const { cache, source } = buildCache();
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const result = await cache.thumbnail(2);
|
||||||
|
expect(source.thumbnailCalls).toEqual([2]);
|
||||||
|
expect(result.path).toBe(join(cacheDir, "thumbnails", "2.jpg"));
|
||||||
|
expect(existsSync(result.path)).toBe(true);
|
||||||
|
expect(cache.pathsFor(2).thumbnailPath).toBe(result.path);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("shares one download between concurrent callers for the same file", async () => {
|
||||||
|
const { cache, source } = buildCache();
|
||||||
|
await cache.open();
|
||||||
|
const gate = deferred();
|
||||||
|
source.gates.set(1, gate.promise);
|
||||||
|
|
||||||
|
const a = cache.original(1);
|
||||||
|
const b = cache.original(1);
|
||||||
|
gate.resolve();
|
||||||
|
const [ra, rb] = await Promise.all([a, b]);
|
||||||
|
|
||||||
|
expect(source.originalCalls).toEqual([1]);
|
||||||
|
expect(ra.path).toBe(rb.path);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not record a path when the fetched file is empty", async () => {
|
||||||
|
const { cache, source } = buildCache();
|
||||||
|
source.emptyFor.add(1);
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
await expect(cache.original(1)).rejects.toThrow(/empty/i);
|
||||||
|
expect(cache.pathsFor(1).originalPath).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects an unknown file", async () => {
|
||||||
|
const { cache } = buildCache({ files: [] });
|
||||||
|
await cache.open();
|
||||||
|
await expect(cache.original(999)).rejects.toThrow(/unknown file/i);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("ContentCache.ensureThumbnails", () => {
|
||||||
|
it("downloads once for a file listed twice and reports every id", async () => {
|
||||||
|
const { cache, source } = buildCache();
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const results = await cache.ensureThumbnails({
|
||||||
|
fileIDs: [1, 1, 2],
|
||||||
|
priority: "visible",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(source.thumbnailCalls.sort()).toEqual([1, 2]);
|
||||||
|
expect(results).toEqual([
|
||||||
|
{ fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") },
|
||||||
|
{ fileID: 2, path: join(cacheDir, "thumbnails", "2.jpg") },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips present files and reports a skipped event", async () => {
|
||||||
|
const thumbnails = join(cacheDir, "thumbnails");
|
||||||
|
mkdirSync(thumbnails, { recursive: true });
|
||||||
|
writeFileSync(join(thumbnails, "1.jpg"), "present");
|
||||||
|
|
||||||
|
const { cache, source } = buildCache();
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const events: EnsureEvent[] = [];
|
||||||
|
const results = await cache.ensureThumbnails({
|
||||||
|
fileIDs: [1, 2],
|
||||||
|
priority: "ahead",
|
||||||
|
onProgress: (e) => events.push(e),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(source.thumbnailCalls).toEqual([2]);
|
||||||
|
expect(results).toEqual([
|
||||||
|
{ fileID: 1, path: join(thumbnails, "1.jpg") },
|
||||||
|
{ fileID: 2, path: join(thumbnails, "2.jpg") },
|
||||||
|
]);
|
||||||
|
expect(events).toContainEqual({
|
||||||
|
fileID: 1,
|
||||||
|
status: "skipped",
|
||||||
|
path: join(thumbnails, "1.jpg"),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("serves a visible request ahead of an already-queued background one", async () => {
|
||||||
|
// One thumbnail slot, so exactly one fetch runs at a time and the rest
|
||||||
|
// wait in the pool. A background fetch takes the slot; a background and
|
||||||
|
// a visible fetch queue behind it. When the slot frees, the pool must
|
||||||
|
// pick the visible (on-demand) request ahead of the background one that
|
||||||
|
// was submitted first. The completion order is the observable.
|
||||||
|
const pools = new RequestPools({ thumbnailConcurrency: 1 });
|
||||||
|
const { cache, source } = buildCache({ pools });
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const gateA = deferred();
|
||||||
|
const gateB = deferred();
|
||||||
|
const gateC = deferred();
|
||||||
|
source.gates.set(1, gateA.promise);
|
||||||
|
source.gates.set(2, gateB.promise);
|
||||||
|
source.gates.set(3, gateC.promise);
|
||||||
|
|
||||||
|
const bgFirst = cache.ensureThumbnails({
|
||||||
|
fileIDs: [1],
|
||||||
|
priority: "background",
|
||||||
|
});
|
||||||
|
// Let fetch 1 take the only slot before the others queue.
|
||||||
|
await Promise.resolve();
|
||||||
|
const bgSecond = cache.ensureThumbnails({
|
||||||
|
fileIDs: [2],
|
||||||
|
priority: "background",
|
||||||
|
});
|
||||||
|
const visible = cache.ensureThumbnails({
|
||||||
|
fileIDs: [3],
|
||||||
|
priority: "visible",
|
||||||
|
});
|
||||||
|
|
||||||
|
gateA.resolve();
|
||||||
|
gateC.resolve();
|
||||||
|
gateB.resolve();
|
||||||
|
await Promise.all([bgFirst, bgSecond, visible]);
|
||||||
|
|
||||||
|
// 1 ran first (it held the slot). Of the two that were queued, the
|
||||||
|
// visible id 3 was served before the background id 2.
|
||||||
|
expect(source.completed).toEqual([1, 3, 2]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops queued work on abort but keeps an in-flight fetch", async () => {
|
||||||
|
const pools = new RequestPools({ thumbnailConcurrency: 1 });
|
||||||
|
const { cache, source } = buildCache({ pools });
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const gate = deferred();
|
||||||
|
source.gates.set(1, gate.promise);
|
||||||
|
const controller = new AbortController();
|
||||||
|
|
||||||
|
const pending = cache.ensureThumbnails({
|
||||||
|
fileIDs: [1, 2],
|
||||||
|
priority: "ahead",
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
// Fetch 1 is in flight (holds the slot); 2 is queued.
|
||||||
|
await Promise.resolve();
|
||||||
|
controller.abort();
|
||||||
|
gate.resolve();
|
||||||
|
|
||||||
|
const results = await pending;
|
||||||
|
|
||||||
|
// The in-flight fetch finished and is kept; the queued one was dropped
|
||||||
|
// before it ran.
|
||||||
|
expect(source.thumbnailCalls).toEqual([1]);
|
||||||
|
expect(results).toEqual([
|
||||||
|
{ fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") },
|
||||||
|
{ fileID: 2, error: "aborted" },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("captures a per-file failure without failing the batch", async () => {
|
||||||
|
const { cache } = buildCache({ files: [file(1)] });
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const results = await cache.ensureThumbnails({
|
||||||
|
fileIDs: [1, 2],
|
||||||
|
priority: "background",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(results[0]).toEqual({
|
||||||
|
fileID: 1,
|
||||||
|
path: join(cacheDir, "thumbnails", "1.jpg"),
|
||||||
|
});
|
||||||
|
expect(results[1]?.fileID).toBe(2);
|
||||||
|
expect(results[1]?.error).toMatch(/unknown file/i);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user