Precache all thumbnails and pinned originals from Library.open (closes #48) #73
+38
-9
@@ -328,12 +328,45 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> {
|
async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> {
|
||||||
const priority = poolPriorityOf(args.priority);
|
return this.ensureMany(
|
||||||
|
"thumbnail",
|
||||||
|
poolPriorityOf(args.priority),
|
||||||
|
args.fileIDs,
|
||||||
|
args.signal,
|
||||||
|
args.onProgress,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fill originals through the content pool for the precache (#48), always at
|
||||||
|
// background priority so an on-demand `original()` preempts the fill. A
|
||||||
|
// present original is a map lookup and no fetch; a per-file failure is
|
||||||
|
// returned, not thrown, so one bad file never halts a background sweep.
|
||||||
|
async ensureOriginals(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
signal?: AbortSignal;
|
||||||
|
onProgress?: (event: EnsureEvent) => void;
|
||||||
|
}): Promise<EnsureResult[]> {
|
||||||
|
return this.ensureMany(
|
||||||
|
"original",
|
||||||
|
"background",
|
||||||
|
args.fileIDs,
|
||||||
|
args.signal,
|
||||||
|
args.onProgress,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async ensureMany(
|
||||||
|
kind: Kind,
|
||||||
|
priority: Priority,
|
||||||
|
fileIDs: number[],
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
onProgress: ((event: EnsureEvent) => void) | undefined,
|
||||||
|
): Promise<EnsureResult[]> {
|
||||||
// Dedup the request list so a repeated fileID is fetched once and
|
// Dedup the request list so a repeated fileID is fetched once and
|
||||||
// reported once, in first-requested order.
|
// reported once, in first-requested order.
|
||||||
const seen = new Set<number>();
|
const seen = new Set<number>();
|
||||||
const unique: number[] = [];
|
const unique: number[] = [];
|
||||||
for (const id of args.fileIDs) {
|
for (const id of fileIDs) {
|
||||||
if (!seen.has(id)) {
|
if (!seen.has(id)) {
|
||||||
seen.add(id);
|
seen.add(id);
|
||||||
unique.push(id);
|
unique.push(id);
|
||||||
@@ -341,24 +374,20 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
}
|
}
|
||||||
return Promise.all(
|
return Promise.all(
|
||||||
unique.map((fileID) =>
|
unique.map((fileID) =>
|
||||||
this.ensureOne(fileID, priority, args.signal, args.onProgress),
|
this.ensureOne(fileID, kind, priority, signal, onProgress),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async ensureOne(
|
private async ensureOne(
|
||||||
fileID: number,
|
fileID: number,
|
||||||
|
kind: Kind,
|
||||||
priority: Priority,
|
priority: Priority,
|
||||||
signal: AbortSignal | undefined,
|
signal: AbortSignal | undefined,
|
||||||
onProgress: ((event: EnsureEvent) => void) | undefined,
|
onProgress: ((event: EnsureEvent) => void) | undefined,
|
||||||
): Promise<EnsureResult> {
|
): Promise<EnsureResult> {
|
||||||
try {
|
try {
|
||||||
const result = await this.acquire(
|
const result = await this.acquire(fileID, kind, priority, signal);
|
||||||
fileID,
|
|
||||||
"thumbnail",
|
|
||||||
priority,
|
|
||||||
signal,
|
|
||||||
);
|
|
||||||
const status = result.cached ? "skipped" : "done";
|
const status = result.cached ? "skipped" : "done";
|
||||||
onProgress?.({ fileID, status, path: result.path });
|
onProgress?.({ fileID, status, path: result.path });
|
||||||
return { fileID, path: result.path };
|
return { fileID, path: result.path };
|
||||||
|
|||||||
+84
-14
@@ -49,6 +49,7 @@ import {
|
|||||||
type EnsureResult,
|
type EnsureResult,
|
||||||
} from "./content.js";
|
} from "./content.js";
|
||||||
import { makeMLDataAPI, type MLDataAPI } from "./mlsearch.js";
|
import { makeMLDataAPI, type MLDataAPI } from "./mlsearch.js";
|
||||||
|
import { Precache } from "./precache.js";
|
||||||
|
|
||||||
export {
|
export {
|
||||||
Album,
|
Album,
|
||||||
@@ -87,6 +88,23 @@ export {
|
|||||||
|
|
||||||
export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3;
|
export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3;
|
||||||
|
|
||||||
|
// Project a metadata store into by-id records, filling each record's cache
|
||||||
|
// paths from the content cache when one is given. Shared by the live read
|
||||||
|
// projection and the precache's initial seeding at open().
|
||||||
|
const deriveRecordsFromStore = (
|
||||||
|
store: MetadataStore,
|
||||||
|
cache?: ContentCache,
|
||||||
|
): DerivedRecords => {
|
||||||
|
const collections = store.listCollections();
|
||||||
|
const files: EnteFile[] = [];
|
||||||
|
for (const c of collections) files.push(...store.listFiles(c.id));
|
||||||
|
return deriveRecords(
|
||||||
|
collections,
|
||||||
|
files,
|
||||||
|
cache ? (fileID) => cache.pathsFor(fileID) : undefined,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
// The slice of `Client` the library depends on. Narrowing to an interface lets
|
// The slice of `Client` the library depends on. Narrowing to an interface lets
|
||||||
// tests drive a mock with no crypto or network; the real `Client` satisfies it
|
// tests drive a mock with no crypto or network; the real `Client` satisfies it
|
||||||
// structurally.
|
// structurally.
|
||||||
@@ -115,9 +133,15 @@ export interface LibraryClient {
|
|||||||
// A progress event for one unit of background work. A metadata "refresh" or an
|
// A progress event for one unit of background work. A metadata "refresh" or an
|
||||||
// ML "fetchMLData" pass each fire "started" before their network work and then
|
// ML "fetchMLData" pass each fire "started" before their network work and then
|
||||||
// exactly one of "done" or "failed"; "failed" carries the error message and
|
// exactly one of "done" or "failed"; "failed" carries the error message and
|
||||||
// an ML "done" reports how many payloads it stored.
|
// an ML "done" reports how many payloads it stored. The precache fills
|
||||||
|
// ("precacheThumbnails"/"precacheOriginals", #48) fire "started"/"done" around
|
||||||
|
// each sweep that has work, "done" reporting the count newly cached.
|
||||||
export interface RefreshEvent {
|
export interface RefreshEvent {
|
||||||
operation: "refresh" | "fetchMLData";
|
operation:
|
||||||
|
| "refresh"
|
||||||
|
| "fetchMLData"
|
||||||
|
| "precacheThumbnails"
|
||||||
|
| "precacheOriginals";
|
||||||
status: "started" | "done" | "failed";
|
status: "started" | "done" | "failed";
|
||||||
error?: string;
|
error?: string;
|
||||||
fetched?: number;
|
fetched?: number;
|
||||||
@@ -147,9 +171,17 @@ export interface LibraryOptions {
|
|||||||
// down as the disk fills; `status().originalsLimitBytes` reports it.
|
// down as the disk fills; `status().originalsLimitBytes` reports it.
|
||||||
cacheOriginalsMaxBytes?: number;
|
cacheOriginalsMaxBytes?: number;
|
||||||
freeBelowBytes?: number;
|
freeBelowBytes?: number;
|
||||||
// Whether an original is pinned and so never evicted (favorites + latest
|
// An extra pinned predicate OR-ed with the precache's own pinned set
|
||||||
// week; the precache unit #48 supplies the set).
|
// (favorites + latest week, #48). Pinned originals are never evicted.
|
||||||
isOriginalPinned?: (fileID: number) => boolean;
|
isOriginalPinned?: (fileID: number) => boolean;
|
||||||
|
// The aggressive local precache (#48), all starting inside `open()` with no
|
||||||
|
// caller input. Thumbnails: every file, newest first, until all are on
|
||||||
|
// disk. Originals: the favorites album then the latest `precacheOriginalsDays`
|
||||||
|
// window (the days ending at the newest file). Both default on; the days
|
||||||
|
// default to 7.
|
||||||
|
precacheThumbnails?: boolean;
|
||||||
|
precacheOriginals?: boolean;
|
||||||
|
precacheOriginalsDays?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LibraryStatus {
|
export interface LibraryStatus {
|
||||||
@@ -175,6 +207,13 @@ export interface LibraryStatus {
|
|||||||
// last write or open; undefined when no content cache is open.
|
// last write or open; undefined when no content cache is open.
|
||||||
originalsUsedBytes?: number;
|
originalsUsedBytes?: number;
|
||||||
originalsLimitBytes?: number;
|
originalsLimitBytes?: number;
|
||||||
|
// Precache progress (#48); undefined when no content cache is open. Totals
|
||||||
|
// are the files targeted (0 when a fill is disabled); "cached" is how many
|
||||||
|
// of them are on disk.
|
||||||
|
thumbnailsCached?: number;
|
||||||
|
thumbnailsTotal?: number;
|
||||||
|
originalsCached?: number;
|
||||||
|
originalsPinned?: number;
|
||||||
closed: boolean;
|
closed: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -207,6 +246,8 @@ export class Library {
|
|||||||
private readonly pools: RequestPools;
|
private readonly pools: RequestPools;
|
||||||
// The ML-data cache, present only when the client can fetch ML data.
|
// The ML-data cache, present only when the client can fetch ML data.
|
||||||
private readonly mlStore?: MLDataStore;
|
private readonly mlStore?: MLDataStore;
|
||||||
|
// The local precache (#48), present only when the content cache is.
|
||||||
|
private readonly precache?: Precache;
|
||||||
|
|
||||||
private timer?: ReturnType<typeof setTimeout>;
|
private timer?: ReturnType<typeof setTimeout>;
|
||||||
private refreshing = false;
|
private refreshing = false;
|
||||||
@@ -240,6 +281,7 @@ export class Library {
|
|||||||
pools: RequestPools;
|
pools: RequestPools;
|
||||||
mldata?: MLDataStore;
|
mldata?: MLDataStore;
|
||||||
cache?: ContentCache;
|
cache?: ContentCache;
|
||||||
|
precache?: Precache;
|
||||||
}) {
|
}) {
|
||||||
this.client = args.client;
|
this.client = args.client;
|
||||||
this.store = args.store;
|
this.store = args.store;
|
||||||
@@ -251,6 +293,7 @@ export class Library {
|
|||||||
this.pools = args.pools;
|
this.pools = args.pools;
|
||||||
this.mlStore = args.mldata;
|
this.mlStore = args.mldata;
|
||||||
this.cache = args.cache;
|
this.cache = args.cache;
|
||||||
|
this.precache = args.precache;
|
||||||
this.lastRecords = this.deriveNow();
|
this.lastRecords = this.deriveNow();
|
||||||
|
|
||||||
// The read namespaces derive fresh from the store on each call, so they
|
// The read namespaces derive fresh from the store on each call, so they
|
||||||
@@ -310,7 +353,21 @@ export class Library {
|
|||||||
// the start and the first refresh raises no spurious path-change diff.
|
// the start and the first refresh raises no spurious path-change diff.
|
||||||
const source = opts.contentSource ?? opts.client.contentSource?.();
|
const source = opts.contentSource ?? opts.client.contentSource?.();
|
||||||
let cache: ContentCache | undefined;
|
let cache: ContentCache | undefined;
|
||||||
|
let precache: Precache | undefined;
|
||||||
if (source) {
|
if (source) {
|
||||||
|
// The precache owns the pinned set (favorites + latest week), which
|
||||||
|
// is the cache's eviction predicate. It is built and seeded from
|
||||||
|
// the loaded store first so the cache can wire `isPinned` to it, and
|
||||||
|
// then bound to the cache it fills. A caller-supplied predicate is
|
||||||
|
// OR-ed in so both survive.
|
||||||
|
precache = new Precache({
|
||||||
|
thumbnails: opts.precacheThumbnails,
|
||||||
|
originals: opts.precacheOriginals,
|
||||||
|
originalsDays: opts.precacheOriginalsDays,
|
||||||
|
onEvent: opts.onProgress,
|
||||||
|
});
|
||||||
|
precache.update(deriveRecordsFromStore(store));
|
||||||
|
const extraPinned = opts.isOriginalPinned;
|
||||||
cache = new ContentCache({
|
cache = new ContentCache({
|
||||||
pools,
|
pools,
|
||||||
source,
|
source,
|
||||||
@@ -319,9 +376,12 @@ export class Library {
|
|||||||
getFile: (fileID) => store.getFileByID(fileID),
|
getFile: (fileID) => store.getFileByID(fileID),
|
||||||
cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes,
|
cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes,
|
||||||
freeBelowBytes: opts.freeBelowBytes,
|
freeBelowBytes: opts.freeBelowBytes,
|
||||||
isPinned: opts.isOriginalPinned,
|
isPinned: (fileID) =>
|
||||||
|
precache!.isPinned(fileID) ||
|
||||||
|
(extraPinned?.(fileID) ?? false),
|
||||||
});
|
});
|
||||||
await cache.open();
|
await cache.open();
|
||||||
|
precache.bind(cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
const lib = new Library({
|
const lib = new Library({
|
||||||
@@ -335,8 +395,15 @@ export class Library {
|
|||||||
pools,
|
pools,
|
||||||
mldata,
|
mldata,
|
||||||
cache,
|
cache,
|
||||||
|
precache,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Start filling from whatever the loaded store already holds; each
|
||||||
|
// refresh below re-kicks with the new files (and retries any that
|
||||||
|
// failed). An empty store starts empty here and fills after its first
|
||||||
|
// refresh.
|
||||||
|
precache?.start();
|
||||||
|
|
||||||
if (store.loadedFromDisk) {
|
if (store.loadedFromDisk) {
|
||||||
// An existing copy already answers reads; refresh in the background
|
// An existing copy already answers reads; refresh in the background
|
||||||
// and start the interval once that first cycle settles.
|
// and start the interval once that first cycle settles.
|
||||||
@@ -396,6 +463,7 @@ export class Library {
|
|||||||
}
|
}
|
||||||
const ml = this.mlStore?.stats();
|
const ml = this.mlStore?.stats();
|
||||||
const originals = this.cache?.originalsStatus();
|
const originals = this.cache?.originalsStatus();
|
||||||
|
const pre = this.precache?.status();
|
||||||
return {
|
return {
|
||||||
userID: this.store.userID,
|
userID: this.store.userID,
|
||||||
collections: collections.length,
|
collections: collections.length,
|
||||||
@@ -408,6 +476,10 @@ export class Library {
|
|||||||
mlIndexed: ml?.indexed,
|
mlIndexed: ml?.indexed,
|
||||||
originalsUsedBytes: originals?.usedBytes,
|
originalsUsedBytes: originals?.usedBytes,
|
||||||
originalsLimitBytes: originals?.limitBytes,
|
originalsLimitBytes: originals?.limitBytes,
|
||||||
|
thumbnailsCached: pre?.thumbnailsCached,
|
||||||
|
thumbnailsTotal: pre?.thumbnailsTotal,
|
||||||
|
originalsCached: pre?.originalsCached,
|
||||||
|
originalsPinned: pre?.originalsPinned,
|
||||||
closed: this.closed,
|
closed: this.closed,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -455,6 +527,7 @@ export class Library {
|
|||||||
// finish; it will not schedule another cycle once closed.
|
// finish; it will not schedule another cycle once closed.
|
||||||
close(): void {
|
close(): void {
|
||||||
this.closed = true;
|
this.closed = true;
|
||||||
|
this.precache?.close();
|
||||||
if (this.timer !== undefined) {
|
if (this.timer !== undefined) {
|
||||||
clearTimeout(this.timer);
|
clearTimeout(this.timer);
|
||||||
this.timer = undefined;
|
this.timer = undefined;
|
||||||
@@ -572,7 +645,12 @@ export class Library {
|
|||||||
if (change) this.notify(change);
|
if (change) this.notify(change);
|
||||||
}
|
}
|
||||||
this.lastRecords = next;
|
this.lastRecords = next;
|
||||||
|
// Recompute the fill orders and pinned set against the new library.
|
||||||
|
this.precache?.update(next);
|
||||||
}
|
}
|
||||||
|
// Re-kick the fills every cycle: a finished sweep starts afresh to pick
|
||||||
|
// up new files and retry any that failed, and a running one is left be.
|
||||||
|
this.precache?.start();
|
||||||
|
|
||||||
// Persist whenever RAM holds changes disk has not accepted — including
|
// Persist whenever RAM holds changes disk has not accepted — including
|
||||||
// changes an earlier cycle staged whose save failed. `unsaved` clears
|
// changes an earlier cycle staged whose save failed. `unsaved` clears
|
||||||
@@ -668,15 +746,7 @@ export class Library {
|
|||||||
// Gather every file membership and project the store into by-id records,
|
// Gather every file membership and project the store into by-id records,
|
||||||
// filling each record's cache paths from the content cache when present.
|
// filling each record's cache paths from the content cache when present.
|
||||||
private deriveNow(): DerivedRecords {
|
private deriveNow(): DerivedRecords {
|
||||||
const collections = this.store.listCollections();
|
return deriveRecordsFromStore(this.store, this.cache);
|
||||||
const files: EnteFile[] = [];
|
|
||||||
for (const c of collections) files.push(...this.store.listFiles(c.id));
|
|
||||||
const cache = this.cache;
|
|
||||||
return deriveRecords(
|
|
||||||
collections,
|
|
||||||
files,
|
|
||||||
cache ? (fileID) => cache.pathsFor(fileID) : undefined,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private notify(change: LibraryChange): void {
|
private notify(change: LibraryChange): void {
|
||||||
|
|||||||
@@ -0,0 +1,265 @@
|
|||||||
|
// The aggressive local precache (issue #48), started from `Library.open` with
|
||||||
|
// no caller input.
|
||||||
|
//
|
||||||
|
// Two background fills run concurrently through the shared request pools (#45):
|
||||||
|
//
|
||||||
|
// - Thumbnails: every file in the account, newest first, through the
|
||||||
|
// thumbnail pool until all are on disk. Never evicted. The pool is the same
|
||||||
|
// one `thumbnails.ensure` uses, so a visible or ahead request always jumps
|
||||||
|
// ahead of this background fill and a fileID both want is fetched once.
|
||||||
|
//
|
||||||
|
// - Originals (the pinned set): through the content pool, the favorites album
|
||||||
|
// first, then every file whose `takenAt` falls in the latest
|
||||||
|
// `originalsDays` window — the days ending at the newest file in the
|
||||||
|
// account. The pinned set is the eviction predicate (#47): a pinned
|
||||||
|
// original is never evicted, and a file that leaves the set (a favorite
|
||||||
|
// removed, or the window moving past it on a later refresh) becomes an
|
||||||
|
// ordinary, evictable original with its bytes left in place.
|
||||||
|
//
|
||||||
|
// Both fills yield to on-demand work: every fetch goes to its pool at
|
||||||
|
// background priority, which the pool serves only after on-demand requests. A
|
||||||
|
// file already cached costs one map lookup (`pathsFor`) and no fetch. Each
|
||||||
|
// sweep is driven in bounded chunks so the pool's waiting queue never grows to
|
||||||
|
// the whole account, keeping on-demand preemption and per-admit cost cheap on a
|
||||||
|
// large library. Failures are not fatal: an uncached file is retried on the
|
||||||
|
// next sweep, which `Library` re-kicks after every refresh.
|
||||||
|
|
||||||
|
import type { EnsureResult } from "./content.js";
|
||||||
|
import type { DerivedRecords } from "./records.js";
|
||||||
|
|
||||||
|
const ONE_DAY_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
export const DEFAULT_PRECACHE_ORIGINALS_DAYS = 7;
|
||||||
|
|
||||||
|
// How many files a sweep submits to a pool before awaiting them. Bounds the
|
||||||
|
// pool's waiting queue so on-demand work is never stuck behind the whole
|
||||||
|
// account; the values track each pool's concurrency (#45).
|
||||||
|
const THUMBNAIL_CHUNK = 25;
|
||||||
|
const ORIGINAL_CHUNK = 5;
|
||||||
|
|
||||||
|
// The precache metrics `Library.status()` surfaces.
|
||||||
|
export interface PrecacheStatus {
|
||||||
|
thumbnailsCached: number;
|
||||||
|
thumbnailsTotal: number;
|
||||||
|
originalsCached: number;
|
||||||
|
originalsPinned: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A background-fill progress event. Each active sweep fires "started" before
|
||||||
|
// its fetches and "done" (with the count newly on disk) after; a sweep with
|
||||||
|
// nothing left to fetch is silent.
|
||||||
|
export interface PrecacheEvent {
|
||||||
|
operation: "precacheThumbnails" | "precacheOriginals";
|
||||||
|
status: "started" | "done";
|
||||||
|
fetched?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The slice of the content cache the precache drives. The real `ContentCache`
|
||||||
|
// satisfies it; tests inject a fake.
|
||||||
|
export interface PrecacheCache {
|
||||||
|
pathsFor(fileID: number): { originalPath?: string; thumbnailPath?: string };
|
||||||
|
ensureThumbnails(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
priority: "background";
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<EnsureResult[]>;
|
||||||
|
ensureOriginals(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<EnsureResult[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PrecacheOptions {
|
||||||
|
// Default true; false disables the fill and, for originals, the pinning.
|
||||||
|
thumbnails?: boolean;
|
||||||
|
originals?: boolean;
|
||||||
|
// The latest-week window length in days; default 7.
|
||||||
|
originalsDays?: number;
|
||||||
|
onEvent?: (event: PrecacheEvent) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class Precache {
|
||||||
|
private readonly doThumbnails: boolean;
|
||||||
|
private readonly doOriginals: boolean;
|
||||||
|
private readonly originalsDays: number;
|
||||||
|
private readonly onEvent?: (event: PrecacheEvent) => void;
|
||||||
|
|
||||||
|
private cache?: PrecacheCache;
|
||||||
|
// Every file in the account, newest first (the thumbnail fill order).
|
||||||
|
private thumbOrder: number[] = [];
|
||||||
|
// The pinned originals in fetch order: favorites first, then the window.
|
||||||
|
private originalsOrder: number[] = [];
|
||||||
|
private pinned = new Set<number>();
|
||||||
|
|
||||||
|
// A sweep runs at most once per fill at a time; a re-kick while one runs is
|
||||||
|
// a no-op, and the next refresh re-kicks after it finishes.
|
||||||
|
private thumbRunning = false;
|
||||||
|
private originalsRunning = false;
|
||||||
|
private readonly aborter = new AbortController();
|
||||||
|
private closed = false;
|
||||||
|
|
||||||
|
constructor(opts: PrecacheOptions = {}) {
|
||||||
|
this.doThumbnails = opts.thumbnails ?? true;
|
||||||
|
this.doOriginals = opts.originals ?? true;
|
||||||
|
this.originalsDays =
|
||||||
|
opts.originalsDays ?? DEFAULT_PRECACHE_ORIGINALS_DAYS;
|
||||||
|
this.onEvent = opts.onEvent;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attach the cache the fills fetch through. `isPinned` works before this is
|
||||||
|
// called, so the cache can be constructed with `isPinned` wired in and then
|
||||||
|
// bound here.
|
||||||
|
bind(cache: PrecacheCache): void {
|
||||||
|
this.cache = cache;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whether an original is pinned (favorites + the latest-week window), the
|
||||||
|
// eviction predicate (#47). False for every file when originals precaching
|
||||||
|
// is disabled.
|
||||||
|
isPinned(fileID: number): boolean {
|
||||||
|
return this.pinned.has(fileID);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recompute the fill orders and the pinned set from the current projection.
|
||||||
|
// Called at open and after every refresh that changes the library.
|
||||||
|
update(records: DerivedRecords): void {
|
||||||
|
const photos = [...records.photos.values()].sort(
|
||||||
|
(a, b) => b.takenAt - a.takenAt || b.fileID - a.fileID,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.thumbOrder = this.doThumbnails ? photos.map((p) => p.fileID) : [];
|
||||||
|
|
||||||
|
const pinned = new Set<number>();
|
||||||
|
const order: number[] = [];
|
||||||
|
if (this.doOriginals) {
|
||||||
|
// Favorites first, in the album's own newest-first order.
|
||||||
|
for (const album of records.albums.values()) {
|
||||||
|
if (album.type !== "favorites") continue;
|
||||||
|
for (const id of album.fileIDs) {
|
||||||
|
if (!pinned.has(id)) {
|
||||||
|
pinned.add(id);
|
||||||
|
order.push(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Then the latest-week window, ending at the newest file. Photos
|
||||||
|
// are newest first, so stop once one falls before the window start.
|
||||||
|
const newest = photos[0];
|
||||||
|
if (newest !== undefined) {
|
||||||
|
const windowStart =
|
||||||
|
newest.takenAt - this.originalsDays * ONE_DAY_MS;
|
||||||
|
for (const p of photos) {
|
||||||
|
if (p.takenAt < windowStart) break;
|
||||||
|
if (!pinned.has(p.fileID)) {
|
||||||
|
pinned.add(p.fileID);
|
||||||
|
order.push(p.fileID);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.pinned = pinned;
|
||||||
|
this.originalsOrder = order;
|
||||||
|
}
|
||||||
|
|
||||||
|
status(): PrecacheStatus {
|
||||||
|
const cache = this.cache;
|
||||||
|
let thumbnailsCached = 0;
|
||||||
|
let originalsCached = 0;
|
||||||
|
if (cache) {
|
||||||
|
for (const id of this.thumbOrder)
|
||||||
|
if (cache.pathsFor(id).thumbnailPath !== undefined)
|
||||||
|
thumbnailsCached++;
|
||||||
|
for (const id of this.originalsOrder)
|
||||||
|
if (cache.pathsFor(id).originalPath !== undefined)
|
||||||
|
originalsCached++;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
thumbnailsCached,
|
||||||
|
thumbnailsTotal: this.thumbOrder.length,
|
||||||
|
originalsCached,
|
||||||
|
originalsPinned: this.originalsOrder.length,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kick both fills. Idempotent: a fill already sweeping is left alone. Safe
|
||||||
|
// to call after every refresh; a finished fill starts a fresh sweep that
|
||||||
|
// picks up new files and retries any that failed before.
|
||||||
|
start(): void {
|
||||||
|
if (this.closed || !this.cache) return;
|
||||||
|
if (this.doThumbnails) this.kickThumbnails();
|
||||||
|
if (this.doOriginals) this.kickOriginals();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop the fills. In-flight fetches are left to settle; queued ones drop.
|
||||||
|
close(): void {
|
||||||
|
this.closed = true;
|
||||||
|
this.aborter.abort();
|
||||||
|
}
|
||||||
|
|
||||||
|
private kickThumbnails(): void {
|
||||||
|
if (this.thumbRunning) return;
|
||||||
|
this.thumbRunning = true;
|
||||||
|
void this.sweep(
|
||||||
|
"precacheThumbnails",
|
||||||
|
() => this.thumbOrder,
|
||||||
|
(id) => this.cache!.pathsFor(id).thumbnailPath !== undefined,
|
||||||
|
THUMBNAIL_CHUNK,
|
||||||
|
(chunk) =>
|
||||||
|
this.cache!.ensureThumbnails({
|
||||||
|
fileIDs: chunk,
|
||||||
|
priority: "background",
|
||||||
|
signal: this.aborter.signal,
|
||||||
|
}),
|
||||||
|
).finally(() => {
|
||||||
|
this.thumbRunning = false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private kickOriginals(): void {
|
||||||
|
if (this.originalsRunning) return;
|
||||||
|
this.originalsRunning = true;
|
||||||
|
void this.sweep(
|
||||||
|
"precacheOriginals",
|
||||||
|
() => this.originalsOrder,
|
||||||
|
(id) => this.cache!.pathsFor(id).originalPath !== undefined,
|
||||||
|
ORIGINAL_CHUNK,
|
||||||
|
(chunk) =>
|
||||||
|
this.cache!.ensureOriginals({
|
||||||
|
fileIDs: chunk,
|
||||||
|
signal: this.aborter.signal,
|
||||||
|
}),
|
||||||
|
).finally(() => {
|
||||||
|
this.originalsRunning = false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// One fill sweep: skip files already on disk (one lookup each), fetch the
|
||||||
|
// rest in bounded chunks, and report progress only when there was work.
|
||||||
|
private async sweep(
|
||||||
|
operation: PrecacheEvent["operation"],
|
||||||
|
order: () => number[],
|
||||||
|
present: (fileID: number) => boolean,
|
||||||
|
chunkSize: number,
|
||||||
|
fetch: (chunk: number[]) => Promise<EnsureResult[]>,
|
||||||
|
): Promise<void> {
|
||||||
|
const todo = order().filter((id) => !present(id));
|
||||||
|
if (todo.length === 0) return;
|
||||||
|
this.emit({ operation, status: "started" });
|
||||||
|
let fetched = 0;
|
||||||
|
for (let i = 0; i < todo.length && !this.closed; i += chunkSize) {
|
||||||
|
const results = await fetch(todo.slice(i, i + chunkSize));
|
||||||
|
for (const r of results) if (r.path !== undefined) fetched++;
|
||||||
|
}
|
||||||
|
this.emit({ operation, status: "done", fetched });
|
||||||
|
}
|
||||||
|
|
||||||
|
private emit(event: PrecacheEvent): void {
|
||||||
|
if (!this.onEvent) return;
|
||||||
|
// A misbehaving callback must not break the fill loop.
|
||||||
|
try {
|
||||||
|
this.onEvent(event);
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -142,6 +142,10 @@ const openLibrary = (source: ContentSource): Promise<Library> =>
|
|||||||
cacheDirectory: join(root, "cache"),
|
cacheDirectory: join(root, "cache"),
|
||||||
contentSource: source,
|
contentSource: source,
|
||||||
refreshIntervalSeconds: 3600,
|
refreshIntervalSeconds: 3600,
|
||||||
|
// These tests count exact fetches; the background precache (#48) would
|
||||||
|
// add its own, so it is off here (it is covered in precache.test.ts).
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const readLedger = (
|
const readLedger = (
|
||||||
|
|||||||
@@ -99,6 +99,10 @@ describe("Library content wiring", () => {
|
|||||||
cacheDirectory: join(root, "cache"),
|
cacheDirectory: join(root, "cache"),
|
||||||
contentSource: source,
|
contentSource: source,
|
||||||
refreshIntervalSeconds: 3600,
|
refreshIntervalSeconds: 3600,
|
||||||
|
// On-demand wiring only; the background precache (#48) is covered
|
||||||
|
// in precache.test.ts and would race the exact-count assertions.
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const photo = lib.photos.byID({ fileID: 1 });
|
const photo = lib.photos.byID({ fileID: 1 });
|
||||||
@@ -122,6 +126,10 @@ describe("Library content wiring", () => {
|
|||||||
cacheDirectory: join(root, "cache"),
|
cacheDirectory: join(root, "cache"),
|
||||||
contentSource: source,
|
contentSource: source,
|
||||||
refreshIntervalSeconds: 3600,
|
refreshIntervalSeconds: 3600,
|
||||||
|
// On-demand wiring only; the background precache (#48) is covered
|
||||||
|
// in precache.test.ts and would race the exact-count assertions.
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const results = await lib.thumbnails.ensure({
|
const results = await lib.thumbnails.ensure({
|
||||||
|
|||||||
@@ -0,0 +1,436 @@
|
|||||||
|
/**
|
||||||
|
* The aggressive local precache (issue #48), driven from `Library.open`.
|
||||||
|
*
|
||||||
|
* Two background fills start with no caller input: every thumbnail in the
|
||||||
|
* account newest first, and the originals of the pinned set (the favorites
|
||||||
|
* album then the latest `precacheOriginalsDays` window ending at the newest
|
||||||
|
* file). Both run through the shared pools at background priority, so on-demand
|
||||||
|
* work always preempts them; both report through `status()`. The pinned set is
|
||||||
|
* the eviction predicate (#47), so a pinned original is never evicted and a
|
||||||
|
* file that leaves the set becomes an ordinary, evictable original.
|
||||||
|
*
|
||||||
|
* The unit tests drive `Precache` against a fake cache that records what it was
|
||||||
|
* asked to fetch (order and kind) with no pool or network; the integration
|
||||||
|
* tests drive the real wiring through `Library.open` with a stub content
|
||||||
|
* source, and the eviction test drives the real `ContentCache`.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync, existsSync, utimesSync } from "node:fs";
|
||||||
|
import { writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { Precache, type PrecacheCache } from "../../src/library/precache.js";
|
||||||
|
import {
|
||||||
|
deriveRecords,
|
||||||
|
type DerivedRecords,
|
||||||
|
} from "../../src/library/records.js";
|
||||||
|
import {
|
||||||
|
ContentCache,
|
||||||
|
type ContentSource,
|
||||||
|
type EnsureResult,
|
||||||
|
type StatFsFn,
|
||||||
|
} from "../../src/library/content.js";
|
||||||
|
import { RequestPools } from "../../src/library/pools.js";
|
||||||
|
import { Library } from "../../src/library/index.js";
|
||||||
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||||
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
||||||
|
|
||||||
|
const DAY_MICROS = 24 * 60 * 60 * 1000 * 1000;
|
||||||
|
|
||||||
|
const collection = (
|
||||||
|
id: number,
|
||||||
|
type: Collection["type"] = "album",
|
||||||
|
): Collection => ({
|
||||||
|
id,
|
||||||
|
ownerID: 1,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
name: `album-${id}`,
|
||||||
|
type,
|
||||||
|
updationTime: 1,
|
||||||
|
isShared: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
// A file whose creationTime (microseconds) places it `daysAgo` days before a
|
||||||
|
// fixed reference instant, so the latest-week window is deterministic.
|
||||||
|
const REFERENCE_MICROS = 1_000 * DAY_MICROS;
|
||||||
|
const file = (id: number, collectionID: number, daysAgo: number): EnteFile => ({
|
||||||
|
id,
|
||||||
|
collectionID,
|
||||||
|
ownerID: 1,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
metadata: {
|
||||||
|
title: `file-${id}.jpg`,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: REFERENCE_MICROS - daysAgo * DAY_MICROS,
|
||||||
|
modificationTime: 0,
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: "aGVhZGVy" },
|
||||||
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
||||||
|
updationTime: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
const records = (
|
||||||
|
collections: Collection[],
|
||||||
|
files: EnteFile[],
|
||||||
|
): DerivedRecords => deriveRecords(collections, files);
|
||||||
|
|
||||||
|
// A fake cache: records every fetch (kind + order) and reports the files it has
|
||||||
|
// stored via `pathsFor`. `presentThumbs`/`presentOriginals` seed already-cached
|
||||||
|
// files so the precache skips them with a single lookup.
|
||||||
|
class FakeCache implements PrecacheCache {
|
||||||
|
readonly thumbFetched: number[] = [];
|
||||||
|
readonly originalFetched: number[] = [];
|
||||||
|
readonly presentThumbs = new Set<number>();
|
||||||
|
readonly presentOriginals = new Set<number>();
|
||||||
|
|
||||||
|
pathsFor(fileID: number): {
|
||||||
|
originalPath?: string;
|
||||||
|
thumbnailPath?: string;
|
||||||
|
} {
|
||||||
|
const out: { originalPath?: string; thumbnailPath?: string } = {};
|
||||||
|
if (this.presentThumbs.has(fileID))
|
||||||
|
out.thumbnailPath = `/thumbs/${fileID}`;
|
||||||
|
if (this.presentOriginals.has(fileID))
|
||||||
|
out.originalPath = `/originals/${fileID}`;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensureThumbnails(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
priority: "background";
|
||||||
|
}): Promise<EnsureResult[]> {
|
||||||
|
return args.fileIDs.map((fileID) => {
|
||||||
|
this.thumbFetched.push(fileID);
|
||||||
|
this.presentThumbs.add(fileID);
|
||||||
|
return { fileID, path: `/thumbs/${fileID}` };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensureOriginals(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
}): Promise<EnsureResult[]> {
|
||||||
|
return args.fileIDs.map((fileID) => {
|
||||||
|
this.originalFetched.push(fileID);
|
||||||
|
this.presentOriginals.add(fileID);
|
||||||
|
return { fileID, path: `/originals/${fileID}` };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve once a predicate holds, polling the microtask queue; fails fast
|
||||||
|
// rather than hanging the suite.
|
||||||
|
const until = async (predicate: () => boolean): Promise<void> => {
|
||||||
|
for (let i = 0; i < 1000; i++) {
|
||||||
|
if (predicate()) return;
|
||||||
|
await new Promise((r) => setTimeout(r, 1));
|
||||||
|
}
|
||||||
|
throw new Error("condition not met in time");
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("Precache unit", () => {
|
||||||
|
it("precaches every thumbnail newest first, skipping present ones", async () => {
|
||||||
|
const cols = [collection(1)];
|
||||||
|
const files = [
|
||||||
|
file(1, 1, 0),
|
||||||
|
file(2, 1, 1),
|
||||||
|
file(3, 1, 2),
|
||||||
|
file(4, 1, 3),
|
||||||
|
];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
cache.presentThumbs.add(3); // already on disk: skipped
|
||||||
|
const pre = new Precache({ originals: false });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
pre.start();
|
||||||
|
|
||||||
|
await until(() => cache.thumbFetched.length === 3);
|
||||||
|
// Newest first (file 1 is newest), file 3 skipped by a lookup.
|
||||||
|
expect(cache.thumbFetched).toEqual([1, 2, 4]);
|
||||||
|
expect(cache.originalFetched).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("pins favorites then the latest-week window and precaches their originals in that order", async () => {
|
||||||
|
const cols = [collection(1), collection(2, "favorites")];
|
||||||
|
// File 10 is an old favorite (30 days old); files 1..3 are within the
|
||||||
|
// 7-day window; file 4 is outside it.
|
||||||
|
const files = [
|
||||||
|
file(1, 1, 0),
|
||||||
|
file(2, 1, 2),
|
||||||
|
file(3, 1, 6),
|
||||||
|
file(4, 1, 20),
|
||||||
|
file(10, 2, 30), // favorite, old
|
||||||
|
];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
pre.start();
|
||||||
|
|
||||||
|
await until(() => cache.originalFetched.length === 4);
|
||||||
|
// Favorite (10) first, then the window newest-first (1, 2, 3). File 4
|
||||||
|
// is outside the window and never pinned.
|
||||||
|
expect(cache.originalFetched).toEqual([10, 1, 2, 3]);
|
||||||
|
expect(pre.isPinned(10)).toBe(true);
|
||||||
|
expect(pre.isPinned(1)).toBe(true);
|
||||||
|
expect(pre.isPinned(4)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops a file from the pinned set when the window moves past it", () => {
|
||||||
|
const cols = [collection(1)];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
pre.bind(cache);
|
||||||
|
|
||||||
|
pre.update(records(cols, [file(1, 1, 0), file(2, 1, 3)]));
|
||||||
|
expect(pre.isPinned(2)).toBe(true);
|
||||||
|
|
||||||
|
// A newer file arrives; the window's newest end moves forward so the
|
||||||
|
// 3-day-old file 2 (now 13 days behind the newest) falls out.
|
||||||
|
pre.update(
|
||||||
|
records(cols, [file(3, 1, -10), file(1, 1, 0), file(2, 1, 3)]),
|
||||||
|
);
|
||||||
|
expect(pre.isPinned(3)).toBe(true);
|
||||||
|
expect(pre.isPinned(2)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports progress through status()", async () => {
|
||||||
|
const cols = [collection(1), collection(2, "favorites")];
|
||||||
|
const files = [file(1, 1, 0), file(2, 1, 1), file(10, 2, 0)];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
|
||||||
|
const before = pre.status();
|
||||||
|
expect(before.thumbnailsTotal).toBe(3);
|
||||||
|
expect(before.thumbnailsCached).toBe(0);
|
||||||
|
expect(before.originalsPinned).toBe(3); // files 1, 2, 10 all in window
|
||||||
|
expect(before.originalsCached).toBe(0);
|
||||||
|
|
||||||
|
pre.start();
|
||||||
|
await until(
|
||||||
|
() =>
|
||||||
|
cache.thumbFetched.length === 3 &&
|
||||||
|
cache.originalFetched.length === 3,
|
||||||
|
);
|
||||||
|
const after = pre.status();
|
||||||
|
expect(after.thumbnailsCached).toBe(3);
|
||||||
|
expect(after.originalsCached).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("honours the disable flags", async () => {
|
||||||
|
const cols = [collection(1)];
|
||||||
|
const files = [file(1, 1, 0)];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ thumbnails: false, originals: false });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
pre.start();
|
||||||
|
|
||||||
|
await new Promise((r) => setTimeout(r, 20));
|
||||||
|
expect(cache.thumbFetched).toEqual([]);
|
||||||
|
expect(cache.originalFetched).toEqual([]);
|
||||||
|
expect(pre.isPinned(1)).toBe(false);
|
||||||
|
expect(pre.status().thumbnailsTotal).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- Integration through the real ContentCache and Library ----
|
||||||
|
|
||||||
|
const enteFile = (id: number, collectionID: number): EnteFile =>
|
||||||
|
file(id, collectionID, 0);
|
||||||
|
|
||||||
|
describe("Precache eviction integration", () => {
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-precache-evict-"));
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never evicts a pinned original the precache put in place", async () => {
|
||||||
|
const cacheDir = join(root, "cache");
|
||||||
|
// File 1 is the favorites album's only file (pinned regardless of age);
|
||||||
|
// files 2 and 3 sit outside the latest-week window, so only file 1 is
|
||||||
|
// pinned. The by-id map serves the bytes for each fetch.
|
||||||
|
const cols = [collection(1), collection(2, "favorites")];
|
||||||
|
const files = [file(1, 2, 30), file(2, 1, 40), file(3, 1, 50)];
|
||||||
|
const byID = new Map<number, EnteFile>(files.map((f) => [f.id, f]));
|
||||||
|
const source: ContentSource = {
|
||||||
|
original: async ({ destination }) => {
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ destination }) => {
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const statfs: StatFsFn = async () => ({
|
||||||
|
bsize: 1,
|
||||||
|
bavail: 1_000_000_000,
|
||||||
|
});
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
const cache = new ContentCache({
|
||||||
|
pools: new RequestPools(),
|
||||||
|
source,
|
||||||
|
cacheDirectory: cacheDir,
|
||||||
|
getFile: (id) => byID.get(id),
|
||||||
|
statfs,
|
||||||
|
cacheOriginalsMaxBytes: 25, // holds two 10-byte originals
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
isPinned: (id) => pre.isPinned(id),
|
||||||
|
});
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
expect(pre.isPinned(1)).toBe(true);
|
||||||
|
expect(pre.isPinned(2)).toBe(false);
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
// Fill three originals; the 25-byte cap forces an eviction on the
|
||||||
|
// third, and the pinned file 1 must survive it even though it is the
|
||||||
|
// least-recently-used.
|
||||||
|
await cache.original(1);
|
||||||
|
utimesSync(join(cacheDir, "originals", "1.jpg"), 1000, 1000); // oldest
|
||||||
|
await cache.original(2);
|
||||||
|
utimesSync(join(cacheDir, "originals", "2.jpg"), 2000, 2000);
|
||||||
|
await cache.original(3);
|
||||||
|
|
||||||
|
expect(existsSync(join(cacheDir, "originals", "1.jpg"))).toBe(true);
|
||||||
|
expect(existsSync(join(cacheDir, "originals", "2.jpg"))).toBe(false);
|
||||||
|
expect(existsSync(join(cacheDir, "originals", "3.jpg"))).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Precache preemption", () => {
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-precache-preempt-"));
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lets an on-demand original preempt the background originals fill", async () => {
|
||||||
|
const byID = new Map<number, EnteFile>([
|
||||||
|
[1, enteFile(1, 1)],
|
||||||
|
[2, enteFile(2, 1)],
|
||||||
|
[3, enteFile(3, 1)],
|
||||||
|
]);
|
||||||
|
const finished: number[] = [];
|
||||||
|
let openGate!: () => void;
|
||||||
|
const gate = new Promise<void>((r) => (openGate = r));
|
||||||
|
let sawFirst!: () => void;
|
||||||
|
const firstStarted = new Promise<void>((r) => (sawFirst = r));
|
||||||
|
let started = 0;
|
||||||
|
const source: ContentSource = {
|
||||||
|
original: async ({ file: f, destination }) => {
|
||||||
|
if (++started === 1) sawFirst();
|
||||||
|
await gate;
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
finished.push(f.id);
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ destination }) => {
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
// One content slot, so file 1 holds it while 2 and 3 wait.
|
||||||
|
const cache = new ContentCache({
|
||||||
|
pools: new RequestPools({ contentConcurrency: 1 }),
|
||||||
|
source,
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
getFile: (id) => byID.get(id),
|
||||||
|
statfs: async () => ({ bsize: 1, bavail: 1_000_000_000 }),
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const pA = cache.ensureOriginals({ fileIDs: [1] }); // background
|
||||||
|
await firstStarted; // file 1 now holds the only slot
|
||||||
|
const pB = cache.original(2); // on-demand, queued behind file 1
|
||||||
|
const pC = cache.ensureOriginals({ fileIDs: [3] }); // background, queued
|
||||||
|
await new Promise((r) => setTimeout(r, 5)); // let both enqueue
|
||||||
|
openGate();
|
||||||
|
await Promise.all([pA, pB, pC]);
|
||||||
|
|
||||||
|
// On-demand file 2 was served before the background file 3.
|
||||||
|
expect(finished).toEqual([1, 2, 3]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Precache through Library.open", () => {
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-precache-lib-"));
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
class MockClient {
|
||||||
|
served = false;
|
||||||
|
whoami(): { email: string; userID: number } {
|
||||||
|
return { email: "u@example.com", userID: 7 };
|
||||||
|
}
|
||||||
|
async collectionsSince(): Promise<CollectionsPage> {
|
||||||
|
if (this.served) return { collections: [], deleted: [], cursor: 1 };
|
||||||
|
this.served = true;
|
||||||
|
return {
|
||||||
|
collections: [collection(1), collection(2, "favorites")],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 1,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
async filesSince(args: { collectionID: number }): Promise<FilesPage> {
|
||||||
|
const files =
|
||||||
|
args.collectionID === 1
|
||||||
|
? [enteFile(1, 1), enteFile(2, 1)]
|
||||||
|
: [enteFile(3, 2)];
|
||||||
|
return { files, deleted: [], cursor: 1 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it("starts both precaches from open() and reports them in status()", async () => {
|
||||||
|
const thumbFetched = new Set<number>();
|
||||||
|
const origFetched = new Set<number>();
|
||||||
|
const source: ContentSource = {
|
||||||
|
original: async ({ file: f, destination }) => {
|
||||||
|
origFetched.add(f.id);
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ file: f, destination }) => {
|
||||||
|
thumbFetched.add(f.id);
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const lib = await Library.open({
|
||||||
|
client: new MockClient(),
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
contentSource: source,
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Every file's thumbnail is precached; the favorite (file 3) and the
|
||||||
|
// week's files (1, 2) all have their originals precached.
|
||||||
|
await until(() => thumbFetched.size === 3 && origFetched.size === 3);
|
||||||
|
const status = lib.status();
|
||||||
|
expect(status.thumbnailsTotal).toBe(3);
|
||||||
|
expect(status.thumbnailsCached).toBe(3);
|
||||||
|
expect(status.originalsPinned).toBe(3);
|
||||||
|
expect(status.originalsCached).toBe(3);
|
||||||
|
lib.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user