Compare commits
2
Commits
2fa1e7c3aa
...
4d400f5a0d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4d400f5a0d | ||
|
|
e8575780e4 |
+38
-9
@@ -328,12 +328,45 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> {
|
async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> {
|
||||||
const priority = poolPriorityOf(args.priority);
|
return this.ensureMany(
|
||||||
|
"thumbnail",
|
||||||
|
poolPriorityOf(args.priority),
|
||||||
|
args.fileIDs,
|
||||||
|
args.signal,
|
||||||
|
args.onProgress,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fill originals through the content pool for the precache (#48), always at
|
||||||
|
// background priority so an on-demand `original()` preempts the fill. A
|
||||||
|
// present original is a map lookup and no fetch; a per-file failure is
|
||||||
|
// returned, not thrown, so one bad file never halts a background sweep.
|
||||||
|
async ensureOriginals(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
signal?: AbortSignal;
|
||||||
|
onProgress?: (event: EnsureEvent) => void;
|
||||||
|
}): Promise<EnsureResult[]> {
|
||||||
|
return this.ensureMany(
|
||||||
|
"original",
|
||||||
|
"background",
|
||||||
|
args.fileIDs,
|
||||||
|
args.signal,
|
||||||
|
args.onProgress,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async ensureMany(
|
||||||
|
kind: Kind,
|
||||||
|
priority: Priority,
|
||||||
|
fileIDs: number[],
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
onProgress: ((event: EnsureEvent) => void) | undefined,
|
||||||
|
): Promise<EnsureResult[]> {
|
||||||
// Dedup the request list so a repeated fileID is fetched once and
|
// Dedup the request list so a repeated fileID is fetched once and
|
||||||
// reported once, in first-requested order.
|
// reported once, in first-requested order.
|
||||||
const seen = new Set<number>();
|
const seen = new Set<number>();
|
||||||
const unique: number[] = [];
|
const unique: number[] = [];
|
||||||
for (const id of args.fileIDs) {
|
for (const id of fileIDs) {
|
||||||
if (!seen.has(id)) {
|
if (!seen.has(id)) {
|
||||||
seen.add(id);
|
seen.add(id);
|
||||||
unique.push(id);
|
unique.push(id);
|
||||||
@@ -341,24 +374,20 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
}
|
}
|
||||||
return Promise.all(
|
return Promise.all(
|
||||||
unique.map((fileID) =>
|
unique.map((fileID) =>
|
||||||
this.ensureOne(fileID, priority, args.signal, args.onProgress),
|
this.ensureOne(fileID, kind, priority, signal, onProgress),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async ensureOne(
|
private async ensureOne(
|
||||||
fileID: number,
|
fileID: number,
|
||||||
|
kind: Kind,
|
||||||
priority: Priority,
|
priority: Priority,
|
||||||
signal: AbortSignal | undefined,
|
signal: AbortSignal | undefined,
|
||||||
onProgress: ((event: EnsureEvent) => void) | undefined,
|
onProgress: ((event: EnsureEvent) => void) | undefined,
|
||||||
): Promise<EnsureResult> {
|
): Promise<EnsureResult> {
|
||||||
try {
|
try {
|
||||||
const result = await this.acquire(
|
const result = await this.acquire(fileID, kind, priority, signal);
|
||||||
fileID,
|
|
||||||
"thumbnail",
|
|
||||||
priority,
|
|
||||||
signal,
|
|
||||||
);
|
|
||||||
const status = result.cached ? "skipped" : "done";
|
const status = result.cached ? "skipped" : "done";
|
||||||
onProgress?.({ fileID, status, path: result.path });
|
onProgress?.({ fileID, status, path: result.path });
|
||||||
return { fileID, path: result.path };
|
return { fileID, path: result.path };
|
||||||
|
|||||||
+96
-18
@@ -48,6 +48,8 @@ import {
|
|||||||
type EnsureOptions,
|
type EnsureOptions,
|
||||||
type EnsureResult,
|
type EnsureResult,
|
||||||
} from "./content.js";
|
} from "./content.js";
|
||||||
|
import { makeMLDataAPI, type MLDataAPI } from "./mlsearch.js";
|
||||||
|
import { Precache } from "./precache.js";
|
||||||
|
|
||||||
export {
|
export {
|
||||||
Album,
|
Album,
|
||||||
@@ -71,6 +73,7 @@ export {
|
|||||||
type EnsureResult,
|
type EnsureResult,
|
||||||
type EnsureEvent,
|
type EnsureEvent,
|
||||||
} from "./content.js";
|
} from "./content.js";
|
||||||
|
export { type MLDataAPI, type SimilarResult } from "./mlsearch.js";
|
||||||
import type { CollectionsPage, FilesPage } from "../client.js";
|
import type { CollectionsPage, FilesPage } from "../client.js";
|
||||||
import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js";
|
import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js";
|
||||||
import type { Collection, EnteFile } from "../model/types.js";
|
import type { Collection, EnteFile } from "../model/types.js";
|
||||||
@@ -85,6 +88,23 @@ export {
|
|||||||
|
|
||||||
export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3;
|
export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3;
|
||||||
|
|
||||||
|
// Project a metadata store into by-id records, filling each record's cache
|
||||||
|
// paths from the content cache when one is given. Shared by the live read
|
||||||
|
// projection and the precache's initial seeding at open().
|
||||||
|
const deriveRecordsFromStore = (
|
||||||
|
store: MetadataStore,
|
||||||
|
cache?: ContentCache,
|
||||||
|
): DerivedRecords => {
|
||||||
|
const collections = store.listCollections();
|
||||||
|
const files: EnteFile[] = [];
|
||||||
|
for (const c of collections) files.push(...store.listFiles(c.id));
|
||||||
|
return deriveRecords(
|
||||||
|
collections,
|
||||||
|
files,
|
||||||
|
cache ? (fileID) => cache.pathsFor(fileID) : undefined,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
// The slice of `Client` the library depends on. Narrowing to an interface lets
|
// The slice of `Client` the library depends on. Narrowing to an interface lets
|
||||||
// tests drive a mock with no crypto or network; the real `Client` satisfies it
|
// tests drive a mock with no crypto or network; the real `Client` satisfies it
|
||||||
// structurally.
|
// structurally.
|
||||||
@@ -113,9 +133,15 @@ export interface LibraryClient {
|
|||||||
// A progress event for one unit of background work. A metadata "refresh" or an
|
// A progress event for one unit of background work. A metadata "refresh" or an
|
||||||
// ML "fetchMLData" pass each fire "started" before their network work and then
|
// ML "fetchMLData" pass each fire "started" before their network work and then
|
||||||
// exactly one of "done" or "failed"; "failed" carries the error message and
|
// exactly one of "done" or "failed"; "failed" carries the error message and
|
||||||
// an ML "done" reports how many payloads it stored.
|
// an ML "done" reports how many payloads it stored. The precache fills
|
||||||
|
// ("precacheThumbnails"/"precacheOriginals", #48) fire "started"/"done" around
|
||||||
|
// each sweep that has work, "done" reporting the count newly cached.
|
||||||
export interface RefreshEvent {
|
export interface RefreshEvent {
|
||||||
operation: "refresh" | "fetchMLData";
|
operation:
|
||||||
|
| "refresh"
|
||||||
|
| "fetchMLData"
|
||||||
|
| "precacheThumbnails"
|
||||||
|
| "precacheOriginals";
|
||||||
status: "started" | "done" | "failed";
|
status: "started" | "done" | "failed";
|
||||||
error?: string;
|
error?: string;
|
||||||
fetched?: number;
|
fetched?: number;
|
||||||
@@ -145,9 +171,17 @@ export interface LibraryOptions {
|
|||||||
// down as the disk fills; `status().originalsLimitBytes` reports it.
|
// down as the disk fills; `status().originalsLimitBytes` reports it.
|
||||||
cacheOriginalsMaxBytes?: number;
|
cacheOriginalsMaxBytes?: number;
|
||||||
freeBelowBytes?: number;
|
freeBelowBytes?: number;
|
||||||
// Whether an original is pinned and so never evicted (favorites + latest
|
// An extra pinned predicate OR-ed with the precache's own pinned set
|
||||||
// week; the precache unit #48 supplies the set).
|
// (favorites + latest week, #48). Pinned originals are never evicted.
|
||||||
isOriginalPinned?: (fileID: number) => boolean;
|
isOriginalPinned?: (fileID: number) => boolean;
|
||||||
|
// The aggressive local precache (#48), all starting inside `open()` with no
|
||||||
|
// caller input. Thumbnails: every file, newest first, until all are on
|
||||||
|
// disk. Originals: the favorites album then the latest `precacheOriginalsDays`
|
||||||
|
// window (the days ending at the newest file). Both default on; the days
|
||||||
|
// default to 7.
|
||||||
|
precacheThumbnails?: boolean;
|
||||||
|
precacheOriginals?: boolean;
|
||||||
|
precacheOriginalsDays?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LibraryStatus {
|
export interface LibraryStatus {
|
||||||
@@ -173,6 +207,13 @@ export interface LibraryStatus {
|
|||||||
// last write or open; undefined when no content cache is open.
|
// last write or open; undefined when no content cache is open.
|
||||||
originalsUsedBytes?: number;
|
originalsUsedBytes?: number;
|
||||||
originalsLimitBytes?: number;
|
originalsLimitBytes?: number;
|
||||||
|
// Precache progress (#48); undefined when no content cache is open. Totals
|
||||||
|
// are the files targeted (0 when a fill is disabled); "cached" is how many
|
||||||
|
// of them are on disk.
|
||||||
|
thumbnailsCached?: number;
|
||||||
|
thumbnailsTotal?: number;
|
||||||
|
originalsCached?: number;
|
||||||
|
originalsPinned?: number;
|
||||||
closed: boolean;
|
closed: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -189,6 +230,10 @@ export class Library {
|
|||||||
// The thumbnail-prefetch surface (issue #46): drives the thumbnail pool
|
// The thumbnail-prefetch surface (issue #46): drives the thumbnail pool
|
||||||
// with priority, dedup, and abort.
|
// with priority, dedup, and abort.
|
||||||
readonly thumbnails: ThumbnailsAPI;
|
readonly thumbnails: ThumbnailsAPI;
|
||||||
|
// The content-similarity search surface over the CLIP index (issue #50).
|
||||||
|
// Present whether or not ML fetching is enabled; with no ML store it
|
||||||
|
// returns empty results.
|
||||||
|
readonly mldata: MLDataAPI;
|
||||||
|
|
||||||
private readonly client: LibraryClient;
|
private readonly client: LibraryClient;
|
||||||
private readonly store: MetadataStore;
|
private readonly store: MetadataStore;
|
||||||
@@ -200,7 +245,9 @@ export class Library {
|
|||||||
private readonly onProgress?: RefreshProgressCallback;
|
private readonly onProgress?: RefreshProgressCallback;
|
||||||
private readonly pools: RequestPools;
|
private readonly pools: RequestPools;
|
||||||
// The ML-data cache, present only when the client can fetch ML data.
|
// The ML-data cache, present only when the client can fetch ML data.
|
||||||
private readonly mldata?: MLDataStore;
|
private readonly mlStore?: MLDataStore;
|
||||||
|
// The local precache (#48), present only when the content cache is.
|
||||||
|
private readonly precache?: Precache;
|
||||||
|
|
||||||
private timer?: ReturnType<typeof setTimeout>;
|
private timer?: ReturnType<typeof setTimeout>;
|
||||||
private refreshing = false;
|
private refreshing = false;
|
||||||
@@ -234,6 +281,7 @@ export class Library {
|
|||||||
pools: RequestPools;
|
pools: RequestPools;
|
||||||
mldata?: MLDataStore;
|
mldata?: MLDataStore;
|
||||||
cache?: ContentCache;
|
cache?: ContentCache;
|
||||||
|
precache?: Precache;
|
||||||
}) {
|
}) {
|
||||||
this.client = args.client;
|
this.client = args.client;
|
||||||
this.store = args.store;
|
this.store = args.store;
|
||||||
@@ -243,8 +291,9 @@ export class Library {
|
|||||||
this.intervalMs = args.intervalMs;
|
this.intervalMs = args.intervalMs;
|
||||||
this.onProgress = args.onProgress;
|
this.onProgress = args.onProgress;
|
||||||
this.pools = args.pools;
|
this.pools = args.pools;
|
||||||
this.mldata = args.mldata;
|
this.mlStore = args.mldata;
|
||||||
this.cache = args.cache;
|
this.cache = args.cache;
|
||||||
|
this.precache = args.precache;
|
||||||
this.lastRecords = this.deriveNow();
|
this.lastRecords = this.deriveNow();
|
||||||
|
|
||||||
// The read namespaces derive fresh from the store on each call, so they
|
// The read namespaces derive fresh from the store on each call, so they
|
||||||
@@ -265,6 +314,8 @@ export class Library {
|
|||||||
return this.cache.ensureThumbnails(opts);
|
return this.cache.ensureThumbnails(opts);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
// Reads the ML store live so results grow as ML data is fetched.
|
||||||
|
this.mldata = makeMLDataAPI(() => this.mlStore);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load the cache and start the refresh loop. With an empty cache the first
|
// Load the cache and start the refresh loop. With an empty cache the first
|
||||||
@@ -302,7 +353,21 @@ export class Library {
|
|||||||
// the start and the first refresh raises no spurious path-change diff.
|
// the start and the first refresh raises no spurious path-change diff.
|
||||||
const source = opts.contentSource ?? opts.client.contentSource?.();
|
const source = opts.contentSource ?? opts.client.contentSource?.();
|
||||||
let cache: ContentCache | undefined;
|
let cache: ContentCache | undefined;
|
||||||
|
let precache: Precache | undefined;
|
||||||
if (source) {
|
if (source) {
|
||||||
|
// The precache owns the pinned set (favorites + latest week), which
|
||||||
|
// is the cache's eviction predicate. It is built and seeded from
|
||||||
|
// the loaded store first so the cache can wire `isPinned` to it, and
|
||||||
|
// then bound to the cache it fills. A caller-supplied predicate is
|
||||||
|
// OR-ed in so both survive.
|
||||||
|
precache = new Precache({
|
||||||
|
thumbnails: opts.precacheThumbnails,
|
||||||
|
originals: opts.precacheOriginals,
|
||||||
|
originalsDays: opts.precacheOriginalsDays,
|
||||||
|
onEvent: opts.onProgress,
|
||||||
|
});
|
||||||
|
precache.update(deriveRecordsFromStore(store));
|
||||||
|
const extraPinned = opts.isOriginalPinned;
|
||||||
cache = new ContentCache({
|
cache = new ContentCache({
|
||||||
pools,
|
pools,
|
||||||
source,
|
source,
|
||||||
@@ -311,9 +376,12 @@ export class Library {
|
|||||||
getFile: (fileID) => store.getFileByID(fileID),
|
getFile: (fileID) => store.getFileByID(fileID),
|
||||||
cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes,
|
cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes,
|
||||||
freeBelowBytes: opts.freeBelowBytes,
|
freeBelowBytes: opts.freeBelowBytes,
|
||||||
isPinned: opts.isOriginalPinned,
|
isPinned: (fileID) =>
|
||||||
|
precache!.isPinned(fileID) ||
|
||||||
|
(extraPinned?.(fileID) ?? false),
|
||||||
});
|
});
|
||||||
await cache.open();
|
await cache.open();
|
||||||
|
precache.bind(cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
const lib = new Library({
|
const lib = new Library({
|
||||||
@@ -327,8 +395,15 @@ export class Library {
|
|||||||
pools,
|
pools,
|
||||||
mldata,
|
mldata,
|
||||||
cache,
|
cache,
|
||||||
|
precache,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Start filling from whatever the loaded store already holds; each
|
||||||
|
// refresh below re-kicks with the new files (and retries any that
|
||||||
|
// failed). An empty store starts empty here and fills after its first
|
||||||
|
// refresh.
|
||||||
|
precache?.start();
|
||||||
|
|
||||||
if (store.loadedFromDisk) {
|
if (store.loadedFromDisk) {
|
||||||
// An existing copy already answers reads; refresh in the background
|
// An existing copy already answers reads; refresh in the background
|
||||||
// and start the interval once that first cycle settles.
|
// and start the interval once that first cycle settles.
|
||||||
@@ -386,8 +461,9 @@ export class Library {
|
|||||||
for (const c of collections) {
|
for (const c of collections) {
|
||||||
files += this.store.listFiles(c.id).length;
|
files += this.store.listFiles(c.id).length;
|
||||||
}
|
}
|
||||||
const ml = this.mldata?.stats();
|
const ml = this.mlStore?.stats();
|
||||||
const originals = this.cache?.originalsStatus();
|
const originals = this.cache?.originalsStatus();
|
||||||
|
const pre = this.precache?.status();
|
||||||
return {
|
return {
|
||||||
userID: this.store.userID,
|
userID: this.store.userID,
|
||||||
collections: collections.length,
|
collections: collections.length,
|
||||||
@@ -400,6 +476,10 @@ export class Library {
|
|||||||
mlIndexed: ml?.indexed,
|
mlIndexed: ml?.indexed,
|
||||||
originalsUsedBytes: originals?.usedBytes,
|
originalsUsedBytes: originals?.usedBytes,
|
||||||
originalsLimitBytes: originals?.limitBytes,
|
originalsLimitBytes: originals?.limitBytes,
|
||||||
|
thumbnailsCached: pre?.thumbnailsCached,
|
||||||
|
thumbnailsTotal: pre?.thumbnailsTotal,
|
||||||
|
originalsCached: pre?.originalsCached,
|
||||||
|
originalsPinned: pre?.originalsPinned,
|
||||||
closed: this.closed,
|
closed: this.closed,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -447,6 +527,7 @@ export class Library {
|
|||||||
// finish; it will not schedule another cycle once closed.
|
// finish; it will not schedule another cycle once closed.
|
||||||
close(): void {
|
close(): void {
|
||||||
this.closed = true;
|
this.closed = true;
|
||||||
|
this.precache?.close();
|
||||||
if (this.timer !== undefined) {
|
if (this.timer !== undefined) {
|
||||||
clearTimeout(this.timer);
|
clearTimeout(this.timer);
|
||||||
this.timer = undefined;
|
this.timer = undefined;
|
||||||
@@ -564,7 +645,12 @@ export class Library {
|
|||||||
if (change) this.notify(change);
|
if (change) this.notify(change);
|
||||||
}
|
}
|
||||||
this.lastRecords = next;
|
this.lastRecords = next;
|
||||||
|
// Recompute the fill orders and pinned set against the new library.
|
||||||
|
this.precache?.update(next);
|
||||||
}
|
}
|
||||||
|
// Re-kick the fills every cycle: a finished sweep starts afresh to pick
|
||||||
|
// up new files and retry any that failed, and a running one is left be.
|
||||||
|
this.precache?.start();
|
||||||
|
|
||||||
// Persist whenever RAM holds changes disk has not accepted — including
|
// Persist whenever RAM holds changes disk has not accepted — including
|
||||||
// changes an earlier cycle staged whose save failed. `unsaved` clears
|
// changes an earlier cycle staged whose save failed. `unsaved` clears
|
||||||
@@ -583,7 +669,7 @@ export class Library {
|
|||||||
// advanced), through the metadata pool, and update the CLIP index. Guarded
|
// advanced), through the metadata pool, and update the CLIP index. Guarded
|
||||||
// so passes never overlap; a failure is reported, not thrown.
|
// so passes never overlap; a failure is reported, not thrown.
|
||||||
private async runMLFetch(): Promise<void> {
|
private async runMLFetch(): Promise<void> {
|
||||||
const mldata = this.mldata;
|
const mldata = this.mlStore;
|
||||||
// Bind so the call keeps the client as its receiver when invoked
|
// Bind so the call keeps the client as its receiver when invoked
|
||||||
// through the pool below.
|
// through the pool below.
|
||||||
const fetchMLData = this.client.fetchMLData?.bind(this.client);
|
const fetchMLData = this.client.fetchMLData?.bind(this.client);
|
||||||
@@ -660,15 +746,7 @@ export class Library {
|
|||||||
// Gather every file membership and project the store into by-id records,
|
// Gather every file membership and project the store into by-id records,
|
||||||
// filling each record's cache paths from the content cache when present.
|
// filling each record's cache paths from the content cache when present.
|
||||||
private deriveNow(): DerivedRecords {
|
private deriveNow(): DerivedRecords {
|
||||||
const collections = this.store.listCollections();
|
return deriveRecordsFromStore(this.store, this.cache);
|
||||||
const files: EnteFile[] = [];
|
|
||||||
for (const c of collections) files.push(...this.store.listFiles(c.id));
|
|
||||||
const cache = this.cache;
|
|
||||||
return deriveRecords(
|
|
||||||
collections,
|
|
||||||
files,
|
|
||||||
cache ? (fileID) => cache.pathsFor(fileID) : undefined,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private notify(change: LibraryChange): void {
|
private notify(change: LibraryChange): void {
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
// The content-similarity search surface over the CLIP index (issue #50).
|
||||||
|
//
|
||||||
|
// This is `lib.mldata`. It answers three questions against the ML-data cache
|
||||||
|
// (#49) without touching the network:
|
||||||
|
//
|
||||||
|
// - `forFile` returns the whole stored payload (face boxes, landmarks,
|
||||||
|
// embedding) for a file, read from disk on demand — the only method here
|
||||||
|
// that touches the disk, and the only one that is async.
|
||||||
|
// - `similar` and `searchByEmbedding` rank fileIDs by cosine similarity over
|
||||||
|
// the packed `Float32Array` index alone. That index (~50k×512) already
|
||||||
|
// lives in RAM, so each query is a plain loop over it and nothing else.
|
||||||
|
//
|
||||||
|
// quak bundles no text encoder (owner-deferred), so `searchByEmbedding` takes
|
||||||
|
// the query vector the caller has produced elsewhere; `similar` uses the
|
||||||
|
// query file's own indexed embedding.
|
||||||
|
|
||||||
|
import type { MLData } from "../mldata-fetch.js";
|
||||||
|
import type { MLDataStore, MLIndex } from "./mldata.js";
|
||||||
|
|
||||||
|
// How many nearest files a query returns when the caller names no limit.
|
||||||
|
const DEFAULT_LIMIT = 20;
|
||||||
|
|
||||||
|
// One ranked result: a fileID and its cosine similarity to the query, in
|
||||||
|
// [-1, 1]. Callers wanting only the ids read `.fileID`.
|
||||||
|
export interface SimilarResult {
|
||||||
|
fileID: number;
|
||||||
|
score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MLDataAPI {
|
||||||
|
// The whole stored ML payload for a file, or undefined when it is not
|
||||||
|
// cached. Reads the payload from disk, so it is async.
|
||||||
|
forFile(args: { fileID: number }): Promise<MLData | undefined>;
|
||||||
|
// The files nearest the given file by cosine over their CLIP embeddings,
|
||||||
|
// most similar first, excluding the file itself. Empty when the file has
|
||||||
|
// no indexed embedding.
|
||||||
|
similar(args: { fileID: number; limit?: number }): SimilarResult[];
|
||||||
|
// The files nearest a caller-supplied query embedding by cosine, most
|
||||||
|
// similar first. Empty when the query is the wrong length for the index,
|
||||||
|
// has zero magnitude, or the index is empty.
|
||||||
|
searchByEmbedding(args: {
|
||||||
|
embedding: ArrayLike<number>;
|
||||||
|
limit?: number;
|
||||||
|
}): SimilarResult[];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rank the packed index by cosine similarity to `query`, most similar first,
|
||||||
|
// and return the top `limit`. `skip` (a query file's own id) is left out. Both
|
||||||
|
// each row's magnitude and the query's are computed here rather than cached:
|
||||||
|
// the index mutates as ML data is fetched, and one plain pass over ~50k×512
|
||||||
|
// floats is fast enough that a norm cache would only add a staleness bug. A
|
||||||
|
// zero-magnitude vector has no direction, so it is dropped rather than divided
|
||||||
|
// by zero.
|
||||||
|
const topByCosine = (
|
||||||
|
index: MLIndex,
|
||||||
|
query: ArrayLike<number>,
|
||||||
|
limit: number,
|
||||||
|
skip?: number,
|
||||||
|
): SimilarResult[] => {
|
||||||
|
const { fileIDs, embeddingLength, embeddings } = index;
|
||||||
|
if (embeddingLength === 0 || query.length !== embeddingLength) return [];
|
||||||
|
|
||||||
|
// Every indexed read below is in range: the inner loops run to
|
||||||
|
// `embeddingLength`, the query is exactly that long (checked above), and
|
||||||
|
// the packed buffer holds `fileIDs.length * embeddingLength` floats.
|
||||||
|
// `noUncheckedIndexedAccess` still widens each read to `number | undefined`,
|
||||||
|
// so they are asserted non-null rather than paying a per-element guard in
|
||||||
|
// this hot ~50k×512 loop.
|
||||||
|
let queryNorm = 0;
|
||||||
|
for (let k = 0; k < embeddingLength; k++) {
|
||||||
|
const q = query[k]!;
|
||||||
|
queryNorm += q * q;
|
||||||
|
}
|
||||||
|
queryNorm = Math.sqrt(queryNorm);
|
||||||
|
if (queryNorm === 0) return [];
|
||||||
|
|
||||||
|
const results: SimilarResult[] = [];
|
||||||
|
for (let i = 0; i < fileIDs.length; i++) {
|
||||||
|
const id = fileIDs[i]!;
|
||||||
|
if (id === skip) continue;
|
||||||
|
const base = i * embeddingLength;
|
||||||
|
let dot = 0;
|
||||||
|
let norm = 0;
|
||||||
|
for (let k = 0; k < embeddingLength; k++) {
|
||||||
|
const v = embeddings[base + k]!;
|
||||||
|
dot += query[k]! * v;
|
||||||
|
norm += v * v;
|
||||||
|
}
|
||||||
|
if (norm === 0) continue;
|
||||||
|
results.push({
|
||||||
|
fileID: id,
|
||||||
|
score: dot / (queryNorm * Math.sqrt(norm)),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Descending score, ties broken by ascending fileID for a stable order.
|
||||||
|
results.sort((a, b) => b.score - a.score || a.fileID - b.fileID);
|
||||||
|
return results.slice(0, Math.max(0, Math.trunc(limit)));
|
||||||
|
};
|
||||||
|
|
||||||
|
// Build the search surface over a store the library supplies lazily (the store
|
||||||
|
// is absent when the client cannot fetch ML data). Reading it per call keeps
|
||||||
|
// the surface current as the index grows.
|
||||||
|
export const makeMLDataAPI = (
|
||||||
|
store: () => MLDataStore | undefined,
|
||||||
|
): MLDataAPI => ({
|
||||||
|
forFile: ({ fileID }): Promise<MLData | undefined> => {
|
||||||
|
const s = store();
|
||||||
|
return s ? s.readPayload(fileID) : Promise.resolve(undefined);
|
||||||
|
},
|
||||||
|
similar: ({ fileID, limit }): SimilarResult[] => {
|
||||||
|
const s = store();
|
||||||
|
if (!s) return [];
|
||||||
|
const index = s.getIndex();
|
||||||
|
const pos = index.fileIDs.indexOf(fileID);
|
||||||
|
if (pos < 0) return [];
|
||||||
|
const base = pos * index.embeddingLength;
|
||||||
|
const query = index.embeddings.subarray(
|
||||||
|
base,
|
||||||
|
base + index.embeddingLength,
|
||||||
|
);
|
||||||
|
return topByCosine(index, query, limit ?? DEFAULT_LIMIT, fileID);
|
||||||
|
},
|
||||||
|
searchByEmbedding: ({ embedding, limit }): SimilarResult[] => {
|
||||||
|
const s = store();
|
||||||
|
if (!s) return [];
|
||||||
|
return topByCosine(s.getIndex(), embedding, limit ?? DEFAULT_LIMIT);
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -0,0 +1,265 @@
|
|||||||
|
// The aggressive local precache (issue #48), started from `Library.open` with
|
||||||
|
// no caller input.
|
||||||
|
//
|
||||||
|
// Two background fills run concurrently through the shared request pools (#45):
|
||||||
|
//
|
||||||
|
// - Thumbnails: every file in the account, newest first, through the
|
||||||
|
// thumbnail pool until all are on disk. Never evicted. The pool is the same
|
||||||
|
// one `thumbnails.ensure` uses, so a visible or ahead request always jumps
|
||||||
|
// ahead of this background fill and a fileID both want is fetched once.
|
||||||
|
//
|
||||||
|
// - Originals (the pinned set): through the content pool, the favorites album
|
||||||
|
// first, then every file whose `takenAt` falls in the latest
|
||||||
|
// `originalsDays` window — the days ending at the newest file in the
|
||||||
|
// account. The pinned set is the eviction predicate (#47): a pinned
|
||||||
|
// original is never evicted, and a file that leaves the set (a favorite
|
||||||
|
// removed, or the window moving past it on a later refresh) becomes an
|
||||||
|
// ordinary, evictable original with its bytes left in place.
|
||||||
|
//
|
||||||
|
// Both fills yield to on-demand work: every fetch goes to its pool at
|
||||||
|
// background priority, which the pool serves only after on-demand requests. A
|
||||||
|
// file already cached costs one map lookup (`pathsFor`) and no fetch. Each
|
||||||
|
// sweep is driven in bounded chunks so the pool's waiting queue never grows to
|
||||||
|
// the whole account, keeping on-demand preemption and per-admit cost cheap on a
|
||||||
|
// large library. Failures are not fatal: an uncached file is retried on the
|
||||||
|
// next sweep, which `Library` re-kicks after every refresh.
|
||||||
|
|
||||||
|
import type { EnsureResult } from "./content.js";
|
||||||
|
import type { DerivedRecords } from "./records.js";
|
||||||
|
|
||||||
|
const ONE_DAY_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
export const DEFAULT_PRECACHE_ORIGINALS_DAYS = 7;
|
||||||
|
|
||||||
|
// How many files a sweep submits to a pool before awaiting them. Bounds the
|
||||||
|
// pool's waiting queue so on-demand work is never stuck behind the whole
|
||||||
|
// account; the values track each pool's concurrency (#45).
|
||||||
|
const THUMBNAIL_CHUNK = 25;
|
||||||
|
const ORIGINAL_CHUNK = 5;
|
||||||
|
|
||||||
|
// The precache metrics `Library.status()` surfaces.
|
||||||
|
export interface PrecacheStatus {
|
||||||
|
thumbnailsCached: number;
|
||||||
|
thumbnailsTotal: number;
|
||||||
|
originalsCached: number;
|
||||||
|
originalsPinned: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A background-fill progress event. Each active sweep fires "started" before
|
||||||
|
// its fetches and "done" (with the count newly on disk) after; a sweep with
|
||||||
|
// nothing left to fetch is silent.
|
||||||
|
export interface PrecacheEvent {
|
||||||
|
operation: "precacheThumbnails" | "precacheOriginals";
|
||||||
|
status: "started" | "done";
|
||||||
|
fetched?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The slice of the content cache the precache drives. The real `ContentCache`
|
||||||
|
// satisfies it; tests inject a fake.
|
||||||
|
export interface PrecacheCache {
|
||||||
|
pathsFor(fileID: number): { originalPath?: string; thumbnailPath?: string };
|
||||||
|
ensureThumbnails(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
priority: "background";
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<EnsureResult[]>;
|
||||||
|
ensureOriginals(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<EnsureResult[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PrecacheOptions {
|
||||||
|
// Default true; false disables the fill and, for originals, the pinning.
|
||||||
|
thumbnails?: boolean;
|
||||||
|
originals?: boolean;
|
||||||
|
// The latest-week window length in days; default 7.
|
||||||
|
originalsDays?: number;
|
||||||
|
onEvent?: (event: PrecacheEvent) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class Precache {
|
||||||
|
private readonly doThumbnails: boolean;
|
||||||
|
private readonly doOriginals: boolean;
|
||||||
|
private readonly originalsDays: number;
|
||||||
|
private readonly onEvent?: (event: PrecacheEvent) => void;
|
||||||
|
|
||||||
|
private cache?: PrecacheCache;
|
||||||
|
// Every file in the account, newest first (the thumbnail fill order).
|
||||||
|
private thumbOrder: number[] = [];
|
||||||
|
// The pinned originals in fetch order: favorites first, then the window.
|
||||||
|
private originalsOrder: number[] = [];
|
||||||
|
private pinned = new Set<number>();
|
||||||
|
|
||||||
|
// A sweep runs at most once per fill at a time; a re-kick while one runs is
|
||||||
|
// a no-op, and the next refresh re-kicks after it finishes.
|
||||||
|
private thumbRunning = false;
|
||||||
|
private originalsRunning = false;
|
||||||
|
private readonly aborter = new AbortController();
|
||||||
|
private closed = false;
|
||||||
|
|
||||||
|
constructor(opts: PrecacheOptions = {}) {
|
||||||
|
this.doThumbnails = opts.thumbnails ?? true;
|
||||||
|
this.doOriginals = opts.originals ?? true;
|
||||||
|
this.originalsDays =
|
||||||
|
opts.originalsDays ?? DEFAULT_PRECACHE_ORIGINALS_DAYS;
|
||||||
|
this.onEvent = opts.onEvent;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attach the cache the fills fetch through. `isPinned` works before this is
|
||||||
|
// called, so the cache can be constructed with `isPinned` wired in and then
|
||||||
|
// bound here.
|
||||||
|
bind(cache: PrecacheCache): void {
|
||||||
|
this.cache = cache;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whether an original is pinned (favorites + the latest-week window), the
|
||||||
|
// eviction predicate (#47). False for every file when originals precaching
|
||||||
|
// is disabled.
|
||||||
|
isPinned(fileID: number): boolean {
|
||||||
|
return this.pinned.has(fileID);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recompute the fill orders and the pinned set from the current projection.
|
||||||
|
// Called at open and after every refresh that changes the library.
|
||||||
|
update(records: DerivedRecords): void {
|
||||||
|
const photos = [...records.photos.values()].sort(
|
||||||
|
(a, b) => b.takenAt - a.takenAt || b.fileID - a.fileID,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.thumbOrder = this.doThumbnails ? photos.map((p) => p.fileID) : [];
|
||||||
|
|
||||||
|
const pinned = new Set<number>();
|
||||||
|
const order: number[] = [];
|
||||||
|
if (this.doOriginals) {
|
||||||
|
// Favorites first, in the album's own newest-first order.
|
||||||
|
for (const album of records.albums.values()) {
|
||||||
|
if (album.type !== "favorites") continue;
|
||||||
|
for (const id of album.fileIDs) {
|
||||||
|
if (!pinned.has(id)) {
|
||||||
|
pinned.add(id);
|
||||||
|
order.push(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Then the latest-week window, ending at the newest file. Photos
|
||||||
|
// are newest first, so stop once one falls before the window start.
|
||||||
|
const newest = photos[0];
|
||||||
|
if (newest !== undefined) {
|
||||||
|
const windowStart =
|
||||||
|
newest.takenAt - this.originalsDays * ONE_DAY_MS;
|
||||||
|
for (const p of photos) {
|
||||||
|
if (p.takenAt < windowStart) break;
|
||||||
|
if (!pinned.has(p.fileID)) {
|
||||||
|
pinned.add(p.fileID);
|
||||||
|
order.push(p.fileID);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.pinned = pinned;
|
||||||
|
this.originalsOrder = order;
|
||||||
|
}
|
||||||
|
|
||||||
|
status(): PrecacheStatus {
|
||||||
|
const cache = this.cache;
|
||||||
|
let thumbnailsCached = 0;
|
||||||
|
let originalsCached = 0;
|
||||||
|
if (cache) {
|
||||||
|
for (const id of this.thumbOrder)
|
||||||
|
if (cache.pathsFor(id).thumbnailPath !== undefined)
|
||||||
|
thumbnailsCached++;
|
||||||
|
for (const id of this.originalsOrder)
|
||||||
|
if (cache.pathsFor(id).originalPath !== undefined)
|
||||||
|
originalsCached++;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
thumbnailsCached,
|
||||||
|
thumbnailsTotal: this.thumbOrder.length,
|
||||||
|
originalsCached,
|
||||||
|
originalsPinned: this.originalsOrder.length,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kick both fills. Idempotent: a fill already sweeping is left alone. Safe
|
||||||
|
// to call after every refresh; a finished fill starts a fresh sweep that
|
||||||
|
// picks up new files and retries any that failed before.
|
||||||
|
start(): void {
|
||||||
|
if (this.closed || !this.cache) return;
|
||||||
|
if (this.doThumbnails) this.kickThumbnails();
|
||||||
|
if (this.doOriginals) this.kickOriginals();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop the fills. In-flight fetches are left to settle; queued ones drop.
|
||||||
|
close(): void {
|
||||||
|
this.closed = true;
|
||||||
|
this.aborter.abort();
|
||||||
|
}
|
||||||
|
|
||||||
|
private kickThumbnails(): void {
|
||||||
|
if (this.thumbRunning) return;
|
||||||
|
this.thumbRunning = true;
|
||||||
|
void this.sweep(
|
||||||
|
"precacheThumbnails",
|
||||||
|
() => this.thumbOrder,
|
||||||
|
(id) => this.cache!.pathsFor(id).thumbnailPath !== undefined,
|
||||||
|
THUMBNAIL_CHUNK,
|
||||||
|
(chunk) =>
|
||||||
|
this.cache!.ensureThumbnails({
|
||||||
|
fileIDs: chunk,
|
||||||
|
priority: "background",
|
||||||
|
signal: this.aborter.signal,
|
||||||
|
}),
|
||||||
|
).finally(() => {
|
||||||
|
this.thumbRunning = false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private kickOriginals(): void {
|
||||||
|
if (this.originalsRunning) return;
|
||||||
|
this.originalsRunning = true;
|
||||||
|
void this.sweep(
|
||||||
|
"precacheOriginals",
|
||||||
|
() => this.originalsOrder,
|
||||||
|
(id) => this.cache!.pathsFor(id).originalPath !== undefined,
|
||||||
|
ORIGINAL_CHUNK,
|
||||||
|
(chunk) =>
|
||||||
|
this.cache!.ensureOriginals({
|
||||||
|
fileIDs: chunk,
|
||||||
|
signal: this.aborter.signal,
|
||||||
|
}),
|
||||||
|
).finally(() => {
|
||||||
|
this.originalsRunning = false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// One fill sweep: skip files already on disk (one lookup each), fetch the
|
||||||
|
// rest in bounded chunks, and report progress only when there was work.
|
||||||
|
private async sweep(
|
||||||
|
operation: PrecacheEvent["operation"],
|
||||||
|
order: () => number[],
|
||||||
|
present: (fileID: number) => boolean,
|
||||||
|
chunkSize: number,
|
||||||
|
fetch: (chunk: number[]) => Promise<EnsureResult[]>,
|
||||||
|
): Promise<void> {
|
||||||
|
const todo = order().filter((id) => !present(id));
|
||||||
|
if (todo.length === 0) return;
|
||||||
|
this.emit({ operation, status: "started" });
|
||||||
|
let fetched = 0;
|
||||||
|
for (let i = 0; i < todo.length && !this.closed; i += chunkSize) {
|
||||||
|
const results = await fetch(todo.slice(i, i + chunkSize));
|
||||||
|
for (const r of results) if (r.path !== undefined) fetched++;
|
||||||
|
}
|
||||||
|
this.emit({ operation, status: "done", fetched });
|
||||||
|
}
|
||||||
|
|
||||||
|
private emit(event: PrecacheEvent): void {
|
||||||
|
if (!this.onEvent) return;
|
||||||
|
// A misbehaving callback must not break the fill loop.
|
||||||
|
try {
|
||||||
|
this.onEvent(event);
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -142,6 +142,10 @@ const openLibrary = (source: ContentSource): Promise<Library> =>
|
|||||||
cacheDirectory: join(root, "cache"),
|
cacheDirectory: join(root, "cache"),
|
||||||
contentSource: source,
|
contentSource: source,
|
||||||
refreshIntervalSeconds: 3600,
|
refreshIntervalSeconds: 3600,
|
||||||
|
// These tests count exact fetches; the background precache (#48) would
|
||||||
|
// add its own, so it is off here (it is covered in precache.test.ts).
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const readLedger = (
|
const readLedger = (
|
||||||
|
|||||||
@@ -99,6 +99,10 @@ describe("Library content wiring", () => {
|
|||||||
cacheDirectory: join(root, "cache"),
|
cacheDirectory: join(root, "cache"),
|
||||||
contentSource: source,
|
contentSource: source,
|
||||||
refreshIntervalSeconds: 3600,
|
refreshIntervalSeconds: 3600,
|
||||||
|
// On-demand wiring only; the background precache (#48) is covered
|
||||||
|
// in precache.test.ts and would race the exact-count assertions.
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const photo = lib.photos.byID({ fileID: 1 });
|
const photo = lib.photos.byID({ fileID: 1 });
|
||||||
@@ -122,6 +126,10 @@ describe("Library content wiring", () => {
|
|||||||
cacheDirectory: join(root, "cache"),
|
cacheDirectory: join(root, "cache"),
|
||||||
contentSource: source,
|
contentSource: source,
|
||||||
refreshIntervalSeconds: 3600,
|
refreshIntervalSeconds: 3600,
|
||||||
|
// On-demand wiring only; the background precache (#48) is covered
|
||||||
|
// in precache.test.ts and would race the exact-count assertions.
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const results = await lib.thumbnails.ensure({
|
const results = await lib.thumbnails.ensure({
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
/**
|
||||||
|
* Tests for the content-similarity search surface over the CLIP index
|
||||||
|
* (issue #50).
|
||||||
|
*
|
||||||
|
* The surface is `lib.mldata`: `forFile` reads the full stored payload from
|
||||||
|
* disk, while `similar` and `searchByEmbedding` rank fileIDs by cosine
|
||||||
|
* similarity over the in-RAM `Float32Array` index alone (no disk, no network).
|
||||||
|
* The fixture uses axis-aligned vectors so the correct cosine ranking is
|
||||||
|
* obvious by inspection; cosine ignores magnitude, so `[2, 0, 0]` ranks above
|
||||||
|
* `[0.8, 0.6, 0]` for a `[1, 0, 0]` query.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { MLDataStore } from "../../src/library/mldata.js";
|
||||||
|
import { makeMLDataAPI, type MLDataAPI } from "../../src/library/mlsearch.js";
|
||||||
|
import type { MLData } from "../../src/mldata-fetch.js";
|
||||||
|
|
||||||
|
// A payload shaped like Ente's: a CLIP embedding plus face data that only the
|
||||||
|
// on-disk payload carries (never the RAM index).
|
||||||
|
const payload = (embedding: number[]): MLData => ({
|
||||||
|
face: { faces: [{ faceID: "f", detection: { box: { x: 0.5 } } }] },
|
||||||
|
clip: { embedding },
|
||||||
|
});
|
||||||
|
|
||||||
|
// A small fixture index. Directions are chosen so every cosine ranking below
|
||||||
|
// is unambiguous.
|
||||||
|
const fixture = (): Map<number, MLData> =>
|
||||||
|
new Map([
|
||||||
|
[10, payload([1, 0, 0])],
|
||||||
|
[20, payload([0.8, 0.6, 0])],
|
||||||
|
[30, payload([0, 1, 0])],
|
||||||
|
[40, payload([-1, 0, 0])],
|
||||||
|
[50, payload([2, 0, 0])],
|
||||||
|
]);
|
||||||
|
|
||||||
|
describe("lib.mldata content-similarity search", () => {
|
||||||
|
let dir: string;
|
||||||
|
let store: MLDataStore;
|
||||||
|
let api: MLDataAPI;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
dir = mkdtempSync(join(tmpdir(), "quak-mlsearch-"));
|
||||||
|
store = await MLDataStore.open(dir);
|
||||||
|
const updation = new Map([...fixture().keys()].map((id) => [id, 1]));
|
||||||
|
await store.storeFetched(fixture(), updation);
|
||||||
|
api = makeMLDataAPI(() => store);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forFile returns the whole stored payload, or undefined when uncached", async () => {
|
||||||
|
const full = await api.forFile({ fileID: 20 });
|
||||||
|
expect(full).toBeDefined();
|
||||||
|
// Face data lives only in the payload, never in the RAM index.
|
||||||
|
expect(full?.face).toBeDefined();
|
||||||
|
expect(full?.clip).toEqual({ embedding: [0.8, 0.6, 0] });
|
||||||
|
expect(await api.forFile({ fileID: 999 })).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("similar ranks other files by cosine and excludes the query itself", () => {
|
||||||
|
// Query is file 10 = [1, 0, 0]. By cosine: 50 (1.0) > 20 (0.8) >
|
||||||
|
// 30 (0) > 40 (-1); 10 itself is left out.
|
||||||
|
const ranked = api.similar({ fileID: 10 });
|
||||||
|
expect(ranked.map((r) => r.fileID)).toEqual([50, 20, 30, 40]);
|
||||||
|
// Cosine ignores magnitude: [2,0,0] is a perfect match for [1,0,0].
|
||||||
|
expect(ranked[0]).toMatchObject({ fileID: 50 });
|
||||||
|
expect(ranked[0].score).toBeCloseTo(1, 5);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("similar honours limit and returns [] for an unindexed file", () => {
|
||||||
|
expect(
|
||||||
|
api.similar({ fileID: 10, limit: 2 }).map((r) => r.fileID),
|
||||||
|
).toEqual([50, 20]);
|
||||||
|
expect(api.similar({ fileID: 999 })).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("searchByEmbedding ranks the index by cosine to the query vector", () => {
|
||||||
|
// Query [0, 1, 0]: 30 (1.0) > 20 (0.6) > {10, 40, 50} all 0, broken by
|
||||||
|
// ascending fileID.
|
||||||
|
const ranked = api.searchByEmbedding({ embedding: [0, 1, 0] });
|
||||||
|
expect(ranked.map((r) => r.fileID)).toEqual([30, 20, 10, 40, 50]);
|
||||||
|
expect(ranked[0].score).toBeCloseTo(1, 5);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
api
|
||||||
|
.searchByEmbedding({ embedding: [0, 1, 0], limit: 2 })
|
||||||
|
.map((r) => r.fileID),
|
||||||
|
).toEqual([30, 20]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("searchByEmbedding returns [] for a wrong-length or zero query", () => {
|
||||||
|
expect(api.searchByEmbedding({ embedding: [1, 0] })).toEqual([]);
|
||||||
|
expect(api.searchByEmbedding({ embedding: [0, 0, 0] })).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("degrades to empty results when no ML store is present", async () => {
|
||||||
|
const none = makeMLDataAPI(() => undefined);
|
||||||
|
expect(await none.forFile({ fileID: 10 })).toBeUndefined();
|
||||||
|
expect(none.similar({ fileID: 10 })).toEqual([]);
|
||||||
|
expect(none.searchByEmbedding({ embedding: [1, 0, 0] })).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,436 @@
|
|||||||
|
/**
|
||||||
|
* The aggressive local precache (issue #48), driven from `Library.open`.
|
||||||
|
*
|
||||||
|
* Two background fills start with no caller input: every thumbnail in the
|
||||||
|
* account newest first, and the originals of the pinned set (the favorites
|
||||||
|
* album then the latest `precacheOriginalsDays` window ending at the newest
|
||||||
|
* file). Both run through the shared pools at background priority, so on-demand
|
||||||
|
* work always preempts them; both report through `status()`. The pinned set is
|
||||||
|
* the eviction predicate (#47), so a pinned original is never evicted and a
|
||||||
|
* file that leaves the set becomes an ordinary, evictable original.
|
||||||
|
*
|
||||||
|
* The unit tests drive `Precache` against a fake cache that records what it was
|
||||||
|
* asked to fetch (order and kind) with no pool or network; the integration
|
||||||
|
* tests drive the real wiring through `Library.open` with a stub content
|
||||||
|
* source, and the eviction test drives the real `ContentCache`.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync, existsSync, utimesSync } from "node:fs";
|
||||||
|
import { writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { Precache, type PrecacheCache } from "../../src/library/precache.js";
|
||||||
|
import {
|
||||||
|
deriveRecords,
|
||||||
|
type DerivedRecords,
|
||||||
|
} from "../../src/library/records.js";
|
||||||
|
import {
|
||||||
|
ContentCache,
|
||||||
|
type ContentSource,
|
||||||
|
type EnsureResult,
|
||||||
|
type StatFsFn,
|
||||||
|
} from "../../src/library/content.js";
|
||||||
|
import { RequestPools } from "../../src/library/pools.js";
|
||||||
|
import { Library } from "../../src/library/index.js";
|
||||||
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||||
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
||||||
|
|
||||||
|
const DAY_MICROS = 24 * 60 * 60 * 1000 * 1000;
|
||||||
|
|
||||||
|
const collection = (
|
||||||
|
id: number,
|
||||||
|
type: Collection["type"] = "album",
|
||||||
|
): Collection => ({
|
||||||
|
id,
|
||||||
|
ownerID: 1,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
name: `album-${id}`,
|
||||||
|
type,
|
||||||
|
updationTime: 1,
|
||||||
|
isShared: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
// A file whose creationTime (microseconds) places it `daysAgo` days before a
|
||||||
|
// fixed reference instant, so the latest-week window is deterministic.
|
||||||
|
const REFERENCE_MICROS = 1_000 * DAY_MICROS;
|
||||||
|
const file = (id: number, collectionID: number, daysAgo: number): EnteFile => ({
|
||||||
|
id,
|
||||||
|
collectionID,
|
||||||
|
ownerID: 1,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
metadata: {
|
||||||
|
title: `file-${id}.jpg`,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: REFERENCE_MICROS - daysAgo * DAY_MICROS,
|
||||||
|
modificationTime: 0,
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: "aGVhZGVy" },
|
||||||
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
||||||
|
updationTime: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
const records = (
|
||||||
|
collections: Collection[],
|
||||||
|
files: EnteFile[],
|
||||||
|
): DerivedRecords => deriveRecords(collections, files);
|
||||||
|
|
||||||
|
// A fake cache: records every fetch (kind + order) and reports the files it has
|
||||||
|
// stored via `pathsFor`. `presentThumbs`/`presentOriginals` seed already-cached
|
||||||
|
// files so the precache skips them with a single lookup.
|
||||||
|
class FakeCache implements PrecacheCache {
|
||||||
|
readonly thumbFetched: number[] = [];
|
||||||
|
readonly originalFetched: number[] = [];
|
||||||
|
readonly presentThumbs = new Set<number>();
|
||||||
|
readonly presentOriginals = new Set<number>();
|
||||||
|
|
||||||
|
pathsFor(fileID: number): {
|
||||||
|
originalPath?: string;
|
||||||
|
thumbnailPath?: string;
|
||||||
|
} {
|
||||||
|
const out: { originalPath?: string; thumbnailPath?: string } = {};
|
||||||
|
if (this.presentThumbs.has(fileID))
|
||||||
|
out.thumbnailPath = `/thumbs/${fileID}`;
|
||||||
|
if (this.presentOriginals.has(fileID))
|
||||||
|
out.originalPath = `/originals/${fileID}`;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensureThumbnails(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
priority: "background";
|
||||||
|
}): Promise<EnsureResult[]> {
|
||||||
|
return args.fileIDs.map((fileID) => {
|
||||||
|
this.thumbFetched.push(fileID);
|
||||||
|
this.presentThumbs.add(fileID);
|
||||||
|
return { fileID, path: `/thumbs/${fileID}` };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensureOriginals(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
}): Promise<EnsureResult[]> {
|
||||||
|
return args.fileIDs.map((fileID) => {
|
||||||
|
this.originalFetched.push(fileID);
|
||||||
|
this.presentOriginals.add(fileID);
|
||||||
|
return { fileID, path: `/originals/${fileID}` };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve once a predicate holds, polling the microtask queue; fails fast
|
||||||
|
// rather than hanging the suite.
|
||||||
|
const until = async (predicate: () => boolean): Promise<void> => {
|
||||||
|
for (let i = 0; i < 1000; i++) {
|
||||||
|
if (predicate()) return;
|
||||||
|
await new Promise((r) => setTimeout(r, 1));
|
||||||
|
}
|
||||||
|
throw new Error("condition not met in time");
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("Precache unit", () => {
|
||||||
|
it("precaches every thumbnail newest first, skipping present ones", async () => {
|
||||||
|
const cols = [collection(1)];
|
||||||
|
const files = [
|
||||||
|
file(1, 1, 0),
|
||||||
|
file(2, 1, 1),
|
||||||
|
file(3, 1, 2),
|
||||||
|
file(4, 1, 3),
|
||||||
|
];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
cache.presentThumbs.add(3); // already on disk: skipped
|
||||||
|
const pre = new Precache({ originals: false });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
pre.start();
|
||||||
|
|
||||||
|
await until(() => cache.thumbFetched.length === 3);
|
||||||
|
// Newest first (file 1 is newest), file 3 skipped by a lookup.
|
||||||
|
expect(cache.thumbFetched).toEqual([1, 2, 4]);
|
||||||
|
expect(cache.originalFetched).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("pins favorites then the latest-week window and precaches their originals in that order", async () => {
|
||||||
|
const cols = [collection(1), collection(2, "favorites")];
|
||||||
|
// File 10 is an old favorite (30 days old); files 1..3 are within the
|
||||||
|
// 7-day window; file 4 is outside it.
|
||||||
|
const files = [
|
||||||
|
file(1, 1, 0),
|
||||||
|
file(2, 1, 2),
|
||||||
|
file(3, 1, 6),
|
||||||
|
file(4, 1, 20),
|
||||||
|
file(10, 2, 30), // favorite, old
|
||||||
|
];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
pre.start();
|
||||||
|
|
||||||
|
await until(() => cache.originalFetched.length === 4);
|
||||||
|
// Favorite (10) first, then the window newest-first (1, 2, 3). File 4
|
||||||
|
// is outside the window and never pinned.
|
||||||
|
expect(cache.originalFetched).toEqual([10, 1, 2, 3]);
|
||||||
|
expect(pre.isPinned(10)).toBe(true);
|
||||||
|
expect(pre.isPinned(1)).toBe(true);
|
||||||
|
expect(pre.isPinned(4)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops a file from the pinned set when the window moves past it", () => {
|
||||||
|
const cols = [collection(1)];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
pre.bind(cache);
|
||||||
|
|
||||||
|
pre.update(records(cols, [file(1, 1, 0), file(2, 1, 3)]));
|
||||||
|
expect(pre.isPinned(2)).toBe(true);
|
||||||
|
|
||||||
|
// A newer file arrives; the window's newest end moves forward so the
|
||||||
|
// 3-day-old file 2 (now 13 days behind the newest) falls out.
|
||||||
|
pre.update(
|
||||||
|
records(cols, [file(3, 1, -10), file(1, 1, 0), file(2, 1, 3)]),
|
||||||
|
);
|
||||||
|
expect(pre.isPinned(3)).toBe(true);
|
||||||
|
expect(pre.isPinned(2)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports progress through status()", async () => {
|
||||||
|
const cols = [collection(1), collection(2, "favorites")];
|
||||||
|
const files = [file(1, 1, 0), file(2, 1, 1), file(10, 2, 0)];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
|
||||||
|
const before = pre.status();
|
||||||
|
expect(before.thumbnailsTotal).toBe(3);
|
||||||
|
expect(before.thumbnailsCached).toBe(0);
|
||||||
|
expect(before.originalsPinned).toBe(3); // files 1, 2, 10 all in window
|
||||||
|
expect(before.originalsCached).toBe(0);
|
||||||
|
|
||||||
|
pre.start();
|
||||||
|
await until(
|
||||||
|
() =>
|
||||||
|
cache.thumbFetched.length === 3 &&
|
||||||
|
cache.originalFetched.length === 3,
|
||||||
|
);
|
||||||
|
const after = pre.status();
|
||||||
|
expect(after.thumbnailsCached).toBe(3);
|
||||||
|
expect(after.originalsCached).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("honours the disable flags", async () => {
|
||||||
|
const cols = [collection(1)];
|
||||||
|
const files = [file(1, 1, 0)];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ thumbnails: false, originals: false });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
pre.start();
|
||||||
|
|
||||||
|
await new Promise((r) => setTimeout(r, 20));
|
||||||
|
expect(cache.thumbFetched).toEqual([]);
|
||||||
|
expect(cache.originalFetched).toEqual([]);
|
||||||
|
expect(pre.isPinned(1)).toBe(false);
|
||||||
|
expect(pre.status().thumbnailsTotal).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- Integration through the real ContentCache and Library ----
|
||||||
|
|
||||||
|
const enteFile = (id: number, collectionID: number): EnteFile =>
|
||||||
|
file(id, collectionID, 0);
|
||||||
|
|
||||||
|
describe("Precache eviction integration", () => {
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-precache-evict-"));
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never evicts a pinned original the precache put in place", async () => {
|
||||||
|
const cacheDir = join(root, "cache");
|
||||||
|
// File 1 is the favorites album's only file (pinned regardless of age);
|
||||||
|
// files 2 and 3 sit outside the latest-week window, so only file 1 is
|
||||||
|
// pinned. The by-id map serves the bytes for each fetch.
|
||||||
|
const cols = [collection(1), collection(2, "favorites")];
|
||||||
|
const files = [file(1, 2, 30), file(2, 1, 40), file(3, 1, 50)];
|
||||||
|
const byID = new Map<number, EnteFile>(files.map((f) => [f.id, f]));
|
||||||
|
const source: ContentSource = {
|
||||||
|
original: async ({ destination }) => {
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ destination }) => {
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const statfs: StatFsFn = async () => ({
|
||||||
|
bsize: 1,
|
||||||
|
bavail: 1_000_000_000,
|
||||||
|
});
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
const cache = new ContentCache({
|
||||||
|
pools: new RequestPools(),
|
||||||
|
source,
|
||||||
|
cacheDirectory: cacheDir,
|
||||||
|
getFile: (id) => byID.get(id),
|
||||||
|
statfs,
|
||||||
|
cacheOriginalsMaxBytes: 25, // holds two 10-byte originals
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
isPinned: (id) => pre.isPinned(id),
|
||||||
|
});
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
expect(pre.isPinned(1)).toBe(true);
|
||||||
|
expect(pre.isPinned(2)).toBe(false);
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
// Fill three originals; the 25-byte cap forces an eviction on the
|
||||||
|
// third, and the pinned file 1 must survive it even though it is the
|
||||||
|
// least-recently-used.
|
||||||
|
await cache.original(1);
|
||||||
|
utimesSync(join(cacheDir, "originals", "1.jpg"), 1000, 1000); // oldest
|
||||||
|
await cache.original(2);
|
||||||
|
utimesSync(join(cacheDir, "originals", "2.jpg"), 2000, 2000);
|
||||||
|
await cache.original(3);
|
||||||
|
|
||||||
|
expect(existsSync(join(cacheDir, "originals", "1.jpg"))).toBe(true);
|
||||||
|
expect(existsSync(join(cacheDir, "originals", "2.jpg"))).toBe(false);
|
||||||
|
expect(existsSync(join(cacheDir, "originals", "3.jpg"))).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Precache preemption", () => {
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-precache-preempt-"));
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lets an on-demand original preempt the background originals fill", async () => {
|
||||||
|
const byID = new Map<number, EnteFile>([
|
||||||
|
[1, enteFile(1, 1)],
|
||||||
|
[2, enteFile(2, 1)],
|
||||||
|
[3, enteFile(3, 1)],
|
||||||
|
]);
|
||||||
|
const finished: number[] = [];
|
||||||
|
let openGate!: () => void;
|
||||||
|
const gate = new Promise<void>((r) => (openGate = r));
|
||||||
|
let sawFirst!: () => void;
|
||||||
|
const firstStarted = new Promise<void>((r) => (sawFirst = r));
|
||||||
|
let started = 0;
|
||||||
|
const source: ContentSource = {
|
||||||
|
original: async ({ file: f, destination }) => {
|
||||||
|
if (++started === 1) sawFirst();
|
||||||
|
await gate;
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
finished.push(f.id);
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ destination }) => {
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
// One content slot, so file 1 holds it while 2 and 3 wait.
|
||||||
|
const cache = new ContentCache({
|
||||||
|
pools: new RequestPools({ contentConcurrency: 1 }),
|
||||||
|
source,
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
getFile: (id) => byID.get(id),
|
||||||
|
statfs: async () => ({ bsize: 1, bavail: 1_000_000_000 }),
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const pA = cache.ensureOriginals({ fileIDs: [1] }); // background
|
||||||
|
await firstStarted; // file 1 now holds the only slot
|
||||||
|
const pB = cache.original(2); // on-demand, queued behind file 1
|
||||||
|
const pC = cache.ensureOriginals({ fileIDs: [3] }); // background, queued
|
||||||
|
await new Promise((r) => setTimeout(r, 5)); // let both enqueue
|
||||||
|
openGate();
|
||||||
|
await Promise.all([pA, pB, pC]);
|
||||||
|
|
||||||
|
// On-demand file 2 was served before the background file 3.
|
||||||
|
expect(finished).toEqual([1, 2, 3]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Precache through Library.open", () => {
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-precache-lib-"));
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
class MockClient {
|
||||||
|
served = false;
|
||||||
|
whoami(): { email: string; userID: number } {
|
||||||
|
return { email: "u@example.com", userID: 7 };
|
||||||
|
}
|
||||||
|
async collectionsSince(): Promise<CollectionsPage> {
|
||||||
|
if (this.served) return { collections: [], deleted: [], cursor: 1 };
|
||||||
|
this.served = true;
|
||||||
|
return {
|
||||||
|
collections: [collection(1), collection(2, "favorites")],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 1,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
async filesSince(args: { collectionID: number }): Promise<FilesPage> {
|
||||||
|
const files =
|
||||||
|
args.collectionID === 1
|
||||||
|
? [enteFile(1, 1), enteFile(2, 1)]
|
||||||
|
: [enteFile(3, 2)];
|
||||||
|
return { files, deleted: [], cursor: 1 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it("starts both precaches from open() and reports them in status()", async () => {
|
||||||
|
const thumbFetched = new Set<number>();
|
||||||
|
const origFetched = new Set<number>();
|
||||||
|
const source: ContentSource = {
|
||||||
|
original: async ({ file: f, destination }) => {
|
||||||
|
origFetched.add(f.id);
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ file: f, destination }) => {
|
||||||
|
thumbFetched.add(f.id);
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const lib = await Library.open({
|
||||||
|
client: new MockClient(),
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
contentSource: source,
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Every file's thumbnail is precached; the favorite (file 3) and the
|
||||||
|
// week's files (1, 2) all have their originals precached.
|
||||||
|
await until(() => thumbFetched.size === 3 && origFetched.size === 3);
|
||||||
|
const status = lib.status();
|
||||||
|
expect(status.thumbnailsTotal).toBe(3);
|
||||||
|
expect(status.thumbnailsCached).toBe(3);
|
||||||
|
expect(status.originalsPinned).toBe(3);
|
||||||
|
expect(status.originalsCached).toBe(3);
|
||||||
|
lib.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user