diff --git a/src/library/content.ts b/src/library/content.ts index 6e76267..2cb19d4 100644 --- a/src/library/content.ts +++ b/src/library/content.ts @@ -328,12 +328,45 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { } async ensureThumbnails(args: EnsureOptions): Promise { - const priority = poolPriorityOf(args.priority); + return this.ensureMany( + "thumbnail", + poolPriorityOf(args.priority), + args.fileIDs, + args.signal, + args.onProgress, + ); + } + + // Fill originals through the content pool for the precache (#48), always at + // background priority so an on-demand `original()` preempts the fill. A + // present original is a map lookup and no fetch; a per-file failure is + // returned, not thrown, so one bad file never halts a background sweep. + async ensureOriginals(args: { + fileIDs: number[]; + signal?: AbortSignal; + onProgress?: (event: EnsureEvent) => void; + }): Promise { + return this.ensureMany( + "original", + "background", + args.fileIDs, + args.signal, + args.onProgress, + ); + } + + private async ensureMany( + kind: Kind, + priority: Priority, + fileIDs: number[], + signal: AbortSignal | undefined, + onProgress: ((event: EnsureEvent) => void) | undefined, + ): Promise { // Dedup the request list so a repeated fileID is fetched once and // reported once, in first-requested order. const seen = new Set(); const unique: number[] = []; - for (const id of args.fileIDs) { + for (const id of fileIDs) { if (!seen.has(id)) { seen.add(id); unique.push(id); @@ -341,24 +374,20 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { } return Promise.all( unique.map((fileID) => - this.ensureOne(fileID, priority, args.signal, args.onProgress), + this.ensureOne(fileID, kind, priority, signal, onProgress), ), ); } private async ensureOne( fileID: number, + kind: Kind, priority: Priority, signal: AbortSignal | undefined, onProgress: ((event: EnsureEvent) => void) | undefined, ): Promise { try { - const result = await this.acquire( - fileID, - "thumbnail", - priority, - signal, - ); + const result = await this.acquire(fileID, kind, priority, signal); const status = result.cached ? "skipped" : "done"; onProgress?.({ fileID, status, path: result.path }); return { fileID, path: result.path }; diff --git a/src/library/index.ts b/src/library/index.ts index e8c1089..f5fd304 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -48,6 +48,7 @@ import { type EnsureOptions, type EnsureResult, } from "./content.js"; +import { Precache } from "./precache.js"; export { Album, @@ -85,6 +86,23 @@ export { export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3; +// Project a metadata store into by-id records, filling each record's cache +// paths from the content cache when one is given. Shared by the live read +// projection and the precache's initial seeding at open(). +const deriveRecordsFromStore = ( + store: MetadataStore, + cache?: ContentCache, +): DerivedRecords => { + const collections = store.listCollections(); + const files: EnteFile[] = []; + for (const c of collections) files.push(...store.listFiles(c.id)); + return deriveRecords( + collections, + files, + cache ? (fileID) => cache.pathsFor(fileID) : undefined, + ); +}; + // The slice of `Client` the library depends on. Narrowing to an interface lets // tests drive a mock with no crypto or network; the real `Client` satisfies it // structurally. @@ -113,9 +131,15 @@ export interface LibraryClient { // A progress event for one unit of background work. A metadata "refresh" or an // ML "fetchMLData" pass each fire "started" before their network work and then // exactly one of "done" or "failed"; "failed" carries the error message and -// an ML "done" reports how many payloads it stored. +// an ML "done" reports how many payloads it stored. The precache fills +// ("precacheThumbnails"/"precacheOriginals", #48) fire "started"/"done" around +// each sweep that has work, "done" reporting the count newly cached. export interface RefreshEvent { - operation: "refresh" | "fetchMLData"; + operation: + | "refresh" + | "fetchMLData" + | "precacheThumbnails" + | "precacheOriginals"; status: "started" | "done" | "failed"; error?: string; fetched?: number; @@ -145,9 +169,17 @@ export interface LibraryOptions { // down as the disk fills; `status().originalsLimitBytes` reports it. cacheOriginalsMaxBytes?: number; freeBelowBytes?: number; - // Whether an original is pinned and so never evicted (favorites + latest - // week; the precache unit #48 supplies the set). + // An extra pinned predicate OR-ed with the precache's own pinned set + // (favorites + latest week, #48). Pinned originals are never evicted. isOriginalPinned?: (fileID: number) => boolean; + // The aggressive local precache (#48), all starting inside `open()` with no + // caller input. Thumbnails: every file, newest first, until all are on + // disk. Originals: the favorites album then the latest `precacheOriginalsDays` + // window (the days ending at the newest file). Both default on; the days + // default to 7. + precacheThumbnails?: boolean; + precacheOriginals?: boolean; + precacheOriginalsDays?: number; } export interface LibraryStatus { @@ -173,6 +205,13 @@ export interface LibraryStatus { // last write or open; undefined when no content cache is open. originalsUsedBytes?: number; originalsLimitBytes?: number; + // Precache progress (#48); undefined when no content cache is open. Totals + // are the files targeted (0 when a fill is disabled); "cached" is how many + // of them are on disk. + thumbnailsCached?: number; + thumbnailsTotal?: number; + originalsCached?: number; + originalsPinned?: number; closed: boolean; } @@ -201,6 +240,8 @@ export class Library { private readonly pools: RequestPools; // The ML-data cache, present only when the client can fetch ML data. private readonly mldata?: MLDataStore; + // The local precache (#48), present only when the content cache is. + private readonly precache?: Precache; private timer?: ReturnType; private refreshing = false; @@ -234,6 +275,7 @@ export class Library { pools: RequestPools; mldata?: MLDataStore; cache?: ContentCache; + precache?: Precache; }) { this.client = args.client; this.store = args.store; @@ -245,6 +287,7 @@ export class Library { this.pools = args.pools; this.mldata = args.mldata; this.cache = args.cache; + this.precache = args.precache; this.lastRecords = this.deriveNow(); // The read namespaces derive fresh from the store on each call, so they @@ -302,7 +345,21 @@ export class Library { // the start and the first refresh raises no spurious path-change diff. const source = opts.contentSource ?? opts.client.contentSource?.(); let cache: ContentCache | undefined; + let precache: Precache | undefined; if (source) { + // The precache owns the pinned set (favorites + latest week), which + // is the cache's eviction predicate. It is built and seeded from + // the loaded store first so the cache can wire `isPinned` to it, and + // then bound to the cache it fills. A caller-supplied predicate is + // OR-ed in so both survive. + precache = new Precache({ + thumbnails: opts.precacheThumbnails, + originals: opts.precacheOriginals, + originalsDays: opts.precacheOriginalsDays, + onEvent: opts.onProgress, + }); + precache.update(deriveRecordsFromStore(store)); + const extraPinned = opts.isOriginalPinned; cache = new ContentCache({ pools, source, @@ -311,9 +368,12 @@ export class Library { getFile: (fileID) => store.getFileByID(fileID), cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes, freeBelowBytes: opts.freeBelowBytes, - isPinned: opts.isOriginalPinned, + isPinned: (fileID) => + precache!.isPinned(fileID) || + (extraPinned?.(fileID) ?? false), }); await cache.open(); + precache.bind(cache); } const lib = new Library({ @@ -327,8 +387,15 @@ export class Library { pools, mldata, cache, + precache, }); + // Start filling from whatever the loaded store already holds; each + // refresh below re-kicks with the new files (and retries any that + // failed). An empty store starts empty here and fills after its first + // refresh. + precache?.start(); + if (store.loadedFromDisk) { // An existing copy already answers reads; refresh in the background // and start the interval once that first cycle settles. @@ -388,6 +455,7 @@ export class Library { } const ml = this.mldata?.stats(); const originals = this.cache?.originalsStatus(); + const pre = this.precache?.status(); return { userID: this.store.userID, collections: collections.length, @@ -400,6 +468,10 @@ export class Library { mlIndexed: ml?.indexed, originalsUsedBytes: originals?.usedBytes, originalsLimitBytes: originals?.limitBytes, + thumbnailsCached: pre?.thumbnailsCached, + thumbnailsTotal: pre?.thumbnailsTotal, + originalsCached: pre?.originalsCached, + originalsPinned: pre?.originalsPinned, closed: this.closed, }; } @@ -447,6 +519,7 @@ export class Library { // finish; it will not schedule another cycle once closed. close(): void { this.closed = true; + this.precache?.close(); if (this.timer !== undefined) { clearTimeout(this.timer); this.timer = undefined; @@ -564,7 +637,12 @@ export class Library { if (change) this.notify(change); } this.lastRecords = next; + // Recompute the fill orders and pinned set against the new library. + this.precache?.update(next); } + // Re-kick the fills every cycle: a finished sweep starts afresh to pick + // up new files and retry any that failed, and a running one is left be. + this.precache?.start(); // Persist whenever RAM holds changes disk has not accepted — including // changes an earlier cycle staged whose save failed. `unsaved` clears @@ -660,15 +738,7 @@ export class Library { // Gather every file membership and project the store into by-id records, // filling each record's cache paths from the content cache when present. private deriveNow(): DerivedRecords { - const collections = this.store.listCollections(); - const files: EnteFile[] = []; - for (const c of collections) files.push(...this.store.listFiles(c.id)); - const cache = this.cache; - return deriveRecords( - collections, - files, - cache ? (fileID) => cache.pathsFor(fileID) : undefined, - ); + return deriveRecordsFromStore(this.store, this.cache); } private notify(change: LibraryChange): void { diff --git a/src/library/precache.ts b/src/library/precache.ts new file mode 100644 index 0000000..f604eaa --- /dev/null +++ b/src/library/precache.ts @@ -0,0 +1,265 @@ +// The aggressive local precache (issue #48), started from `Library.open` with +// no caller input. +// +// Two background fills run concurrently through the shared request pools (#45): +// +// - Thumbnails: every file in the account, newest first, through the +// thumbnail pool until all are on disk. Never evicted. The pool is the same +// one `thumbnails.ensure` uses, so a visible or ahead request always jumps +// ahead of this background fill and a fileID both want is fetched once. +// +// - Originals (the pinned set): through the content pool, the favorites album +// first, then every file whose `takenAt` falls in the latest +// `originalsDays` window — the days ending at the newest file in the +// account. The pinned set is the eviction predicate (#47): a pinned +// original is never evicted, and a file that leaves the set (a favorite +// removed, or the window moving past it on a later refresh) becomes an +// ordinary, evictable original with its bytes left in place. +// +// Both fills yield to on-demand work: every fetch goes to its pool at +// background priority, which the pool serves only after on-demand requests. A +// file already cached costs one map lookup (`pathsFor`) and no fetch. Each +// sweep is driven in bounded chunks so the pool's waiting queue never grows to +// the whole account, keeping on-demand preemption and per-admit cost cheap on a +// large library. Failures are not fatal: an uncached file is retried on the +// next sweep, which `Library` re-kicks after every refresh. + +import type { EnsureResult } from "./content.js"; +import type { DerivedRecords } from "./records.js"; + +const ONE_DAY_MS = 24 * 60 * 60 * 1000; + +export const DEFAULT_PRECACHE_ORIGINALS_DAYS = 7; + +// How many files a sweep submits to a pool before awaiting them. Bounds the +// pool's waiting queue so on-demand work is never stuck behind the whole +// account; the values track each pool's concurrency (#45). +const THUMBNAIL_CHUNK = 25; +const ORIGINAL_CHUNK = 5; + +// The precache metrics `Library.status()` surfaces. +export interface PrecacheStatus { + thumbnailsCached: number; + thumbnailsTotal: number; + originalsCached: number; + originalsPinned: number; +} + +// A background-fill progress event. Each active sweep fires "started" before +// its fetches and "done" (with the count newly on disk) after; a sweep with +// nothing left to fetch is silent. +export interface PrecacheEvent { + operation: "precacheThumbnails" | "precacheOriginals"; + status: "started" | "done"; + fetched?: number; +} + +// The slice of the content cache the precache drives. The real `ContentCache` +// satisfies it; tests inject a fake. +export interface PrecacheCache { + pathsFor(fileID: number): { originalPath?: string; thumbnailPath?: string }; + ensureThumbnails(args: { + fileIDs: number[]; + priority: "background"; + signal?: AbortSignal; + }): Promise; + ensureOriginals(args: { + fileIDs: number[]; + signal?: AbortSignal; + }): Promise; +} + +export interface PrecacheOptions { + // Default true; false disables the fill and, for originals, the pinning. + thumbnails?: boolean; + originals?: boolean; + // The latest-week window length in days; default 7. + originalsDays?: number; + onEvent?: (event: PrecacheEvent) => void; +} + +export class Precache { + private readonly doThumbnails: boolean; + private readonly doOriginals: boolean; + private readonly originalsDays: number; + private readonly onEvent?: (event: PrecacheEvent) => void; + + private cache?: PrecacheCache; + // Every file in the account, newest first (the thumbnail fill order). + private thumbOrder: number[] = []; + // The pinned originals in fetch order: favorites first, then the window. + private originalsOrder: number[] = []; + private pinned = new Set(); + + // A sweep runs at most once per fill at a time; a re-kick while one runs is + // a no-op, and the next refresh re-kicks after it finishes. + private thumbRunning = false; + private originalsRunning = false; + private readonly aborter = new AbortController(); + private closed = false; + + constructor(opts: PrecacheOptions = {}) { + this.doThumbnails = opts.thumbnails ?? true; + this.doOriginals = opts.originals ?? true; + this.originalsDays = + opts.originalsDays ?? DEFAULT_PRECACHE_ORIGINALS_DAYS; + this.onEvent = opts.onEvent; + } + + // Attach the cache the fills fetch through. `isPinned` works before this is + // called, so the cache can be constructed with `isPinned` wired in and then + // bound here. + bind(cache: PrecacheCache): void { + this.cache = cache; + } + + // Whether an original is pinned (favorites + the latest-week window), the + // eviction predicate (#47). False for every file when originals precaching + // is disabled. + isPinned(fileID: number): boolean { + return this.pinned.has(fileID); + } + + // Recompute the fill orders and the pinned set from the current projection. + // Called at open and after every refresh that changes the library. + update(records: DerivedRecords): void { + const photos = [...records.photos.values()].sort( + (a, b) => b.takenAt - a.takenAt || b.fileID - a.fileID, + ); + + this.thumbOrder = this.doThumbnails ? photos.map((p) => p.fileID) : []; + + const pinned = new Set(); + const order: number[] = []; + if (this.doOriginals) { + // Favorites first, in the album's own newest-first order. + for (const album of records.albums.values()) { + if (album.type !== "favorites") continue; + for (const id of album.fileIDs) { + if (!pinned.has(id)) { + pinned.add(id); + order.push(id); + } + } + } + // Then the latest-week window, ending at the newest file. Photos + // are newest first, so stop once one falls before the window start. + const newest = photos[0]; + if (newest !== undefined) { + const windowStart = + newest.takenAt - this.originalsDays * ONE_DAY_MS; + for (const p of photos) { + if (p.takenAt < windowStart) break; + if (!pinned.has(p.fileID)) { + pinned.add(p.fileID); + order.push(p.fileID); + } + } + } + } + this.pinned = pinned; + this.originalsOrder = order; + } + + status(): PrecacheStatus { + const cache = this.cache; + let thumbnailsCached = 0; + let originalsCached = 0; + if (cache) { + for (const id of this.thumbOrder) + if (cache.pathsFor(id).thumbnailPath !== undefined) + thumbnailsCached++; + for (const id of this.originalsOrder) + if (cache.pathsFor(id).originalPath !== undefined) + originalsCached++; + } + return { + thumbnailsCached, + thumbnailsTotal: this.thumbOrder.length, + originalsCached, + originalsPinned: this.originalsOrder.length, + }; + } + + // Kick both fills. Idempotent: a fill already sweeping is left alone. Safe + // to call after every refresh; a finished fill starts a fresh sweep that + // picks up new files and retries any that failed before. + start(): void { + if (this.closed || !this.cache) return; + if (this.doThumbnails) this.kickThumbnails(); + if (this.doOriginals) this.kickOriginals(); + } + + // Stop the fills. In-flight fetches are left to settle; queued ones drop. + close(): void { + this.closed = true; + this.aborter.abort(); + } + + private kickThumbnails(): void { + if (this.thumbRunning) return; + this.thumbRunning = true; + void this.sweep( + "precacheThumbnails", + () => this.thumbOrder, + (id) => this.cache!.pathsFor(id).thumbnailPath !== undefined, + THUMBNAIL_CHUNK, + (chunk) => + this.cache!.ensureThumbnails({ + fileIDs: chunk, + priority: "background", + signal: this.aborter.signal, + }), + ).finally(() => { + this.thumbRunning = false; + }); + } + + private kickOriginals(): void { + if (this.originalsRunning) return; + this.originalsRunning = true; + void this.sweep( + "precacheOriginals", + () => this.originalsOrder, + (id) => this.cache!.pathsFor(id).originalPath !== undefined, + ORIGINAL_CHUNK, + (chunk) => + this.cache!.ensureOriginals({ + fileIDs: chunk, + signal: this.aborter.signal, + }), + ).finally(() => { + this.originalsRunning = false; + }); + } + + // One fill sweep: skip files already on disk (one lookup each), fetch the + // rest in bounded chunks, and report progress only when there was work. + private async sweep( + operation: PrecacheEvent["operation"], + order: () => number[], + present: (fileID: number) => boolean, + chunkSize: number, + fetch: (chunk: number[]) => Promise, + ): Promise { + const todo = order().filter((id) => !present(id)); + if (todo.length === 0) return; + this.emit({ operation, status: "started" }); + let fetched = 0; + for (let i = 0; i < todo.length && !this.closed; i += chunkSize) { + const results = await fetch(todo.slice(i, i + chunkSize)); + for (const r of results) if (r.path !== undefined) fetched++; + } + this.emit({ operation, status: "done", fetched }); + } + + private emit(event: PrecacheEvent): void { + if (!this.onEvent) return; + // A misbehaving callback must not break the fill loop. + try { + this.onEvent(event); + } catch { + // ignore + } + } +} diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 1fb95a3..7823730 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -142,6 +142,10 @@ const openLibrary = (source: ContentSource): Promise => cacheDirectory: join(root, "cache"), contentSource: source, refreshIntervalSeconds: 3600, + // These tests count exact fetches; the background precache (#48) would + // add its own, so it is off here (it is covered in precache.test.ts). + precacheThumbnails: false, + precacheOriginals: false, }); const readLedger = ( diff --git a/test/library/content-library.test.ts b/test/library/content-library.test.ts index e1eb1da..a89f47c 100644 --- a/test/library/content-library.test.ts +++ b/test/library/content-library.test.ts @@ -99,6 +99,10 @@ describe("Library content wiring", () => { cacheDirectory: join(root, "cache"), contentSource: source, refreshIntervalSeconds: 3600, + // On-demand wiring only; the background precache (#48) is covered + // in precache.test.ts and would race the exact-count assertions. + precacheThumbnails: false, + precacheOriginals: false, }); const photo = lib.photos.byID({ fileID: 1 }); @@ -122,6 +126,10 @@ describe("Library content wiring", () => { cacheDirectory: join(root, "cache"), contentSource: source, refreshIntervalSeconds: 3600, + // On-demand wiring only; the background precache (#48) is covered + // in precache.test.ts and would race the exact-count assertions. + precacheThumbnails: false, + precacheOriginals: false, }); const results = await lib.thumbnails.ensure({ diff --git a/test/library/precache.test.ts b/test/library/precache.test.ts new file mode 100644 index 0000000..2580579 --- /dev/null +++ b/test/library/precache.test.ts @@ -0,0 +1,436 @@ +/** + * The aggressive local precache (issue #48), driven from `Library.open`. + * + * Two background fills start with no caller input: every thumbnail in the + * account newest first, and the originals of the pinned set (the favorites + * album then the latest `precacheOriginalsDays` window ending at the newest + * file). Both run through the shared pools at background priority, so on-demand + * work always preempts them; both report through `status()`. The pinned set is + * the eviction predicate (#47), so a pinned original is never evicted and a + * file that leaves the set becomes an ordinary, evictable original. + * + * The unit tests drive `Precache` against a fake cache that records what it was + * asked to fetch (order and kind) with no pool or network; the integration + * tests drive the real wiring through `Library.open` with a stub content + * source, and the eviction test drives the real `ContentCache`. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, existsSync, utimesSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Precache, type PrecacheCache } from "../../src/library/precache.js"; +import { + deriveRecords, + type DerivedRecords, +} from "../../src/library/records.js"; +import { + ContentCache, + type ContentSource, + type EnsureResult, + type StatFsFn, +} from "../../src/library/content.js"; +import { RequestPools } from "../../src/library/pools.js"; +import { Library } from "../../src/library/index.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; + +const DAY_MICROS = 24 * 60 * 60 * 1000 * 1000; + +const collection = ( + id: number, + type: Collection["type"] = "album", +): Collection => ({ + id, + ownerID: 1, + key: new Uint8Array([id & 0xff]), + name: `album-${id}`, + type, + updationTime: 1, + isShared: false, +}); + +// A file whose creationTime (microseconds) places it `daysAgo` days before a +// fixed reference instant, so the latest-week window is deterministic. +const REFERENCE_MICROS = 1_000 * DAY_MICROS; +const file = (id: number, collectionID: number, daysAgo: number): EnteFile => ({ + id, + collectionID, + ownerID: 1, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: REFERENCE_MICROS - daysAgo * DAY_MICROS, + modificationTime: 0, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 1, +}); + +const records = ( + collections: Collection[], + files: EnteFile[], +): DerivedRecords => deriveRecords(collections, files); + +// A fake cache: records every fetch (kind + order) and reports the files it has +// stored via `pathsFor`. `presentThumbs`/`presentOriginals` seed already-cached +// files so the precache skips them with a single lookup. +class FakeCache implements PrecacheCache { + readonly thumbFetched: number[] = []; + readonly originalFetched: number[] = []; + readonly presentThumbs = new Set(); + readonly presentOriginals = new Set(); + + pathsFor(fileID: number): { + originalPath?: string; + thumbnailPath?: string; + } { + const out: { originalPath?: string; thumbnailPath?: string } = {}; + if (this.presentThumbs.has(fileID)) + out.thumbnailPath = `/thumbs/${fileID}`; + if (this.presentOriginals.has(fileID)) + out.originalPath = `/originals/${fileID}`; + return out; + } + + async ensureThumbnails(args: { + fileIDs: number[]; + priority: "background"; + }): Promise { + return args.fileIDs.map((fileID) => { + this.thumbFetched.push(fileID); + this.presentThumbs.add(fileID); + return { fileID, path: `/thumbs/${fileID}` }; + }); + } + + async ensureOriginals(args: { + fileIDs: number[]; + }): Promise { + return args.fileIDs.map((fileID) => { + this.originalFetched.push(fileID); + this.presentOriginals.add(fileID); + return { fileID, path: `/originals/${fileID}` }; + }); + } +} + +// Resolve once a predicate holds, polling the microtask queue; fails fast +// rather than hanging the suite. +const until = async (predicate: () => boolean): Promise => { + for (let i = 0; i < 1000; i++) { + if (predicate()) return; + await new Promise((r) => setTimeout(r, 1)); + } + throw new Error("condition not met in time"); +}; + +describe("Precache unit", () => { + it("precaches every thumbnail newest first, skipping present ones", async () => { + const cols = [collection(1)]; + const files = [ + file(1, 1, 0), + file(2, 1, 1), + file(3, 1, 2), + file(4, 1, 3), + ]; + const cache = new FakeCache(); + cache.presentThumbs.add(3); // already on disk: skipped + const pre = new Precache({ originals: false }); + pre.bind(cache); + pre.update(records(cols, files)); + pre.start(); + + await until(() => cache.thumbFetched.length === 3); + // Newest first (file 1 is newest), file 3 skipped by a lookup. + expect(cache.thumbFetched).toEqual([1, 2, 4]); + expect(cache.originalFetched).toEqual([]); + }); + + it("pins favorites then the latest-week window and precaches their originals in that order", async () => { + const cols = [collection(1), collection(2, "favorites")]; + // File 10 is an old favorite (30 days old); files 1..3 are within the + // 7-day window; file 4 is outside it. + const files = [ + file(1, 1, 0), + file(2, 1, 2), + file(3, 1, 6), + file(4, 1, 20), + file(10, 2, 30), // favorite, old + ]; + const cache = new FakeCache(); + const pre = new Precache({ originalsDays: 7 }); + pre.bind(cache); + pre.update(records(cols, files)); + pre.start(); + + await until(() => cache.originalFetched.length === 4); + // Favorite (10) first, then the window newest-first (1, 2, 3). File 4 + // is outside the window and never pinned. + expect(cache.originalFetched).toEqual([10, 1, 2, 3]); + expect(pre.isPinned(10)).toBe(true); + expect(pre.isPinned(1)).toBe(true); + expect(pre.isPinned(4)).toBe(false); + }); + + it("drops a file from the pinned set when the window moves past it", () => { + const cols = [collection(1)]; + const cache = new FakeCache(); + const pre = new Precache({ originalsDays: 7 }); + pre.bind(cache); + + pre.update(records(cols, [file(1, 1, 0), file(2, 1, 3)])); + expect(pre.isPinned(2)).toBe(true); + + // A newer file arrives; the window's newest end moves forward so the + // 3-day-old file 2 (now 13 days behind the newest) falls out. + pre.update( + records(cols, [file(3, 1, -10), file(1, 1, 0), file(2, 1, 3)]), + ); + expect(pre.isPinned(3)).toBe(true); + expect(pre.isPinned(2)).toBe(false); + }); + + it("reports progress through status()", async () => { + const cols = [collection(1), collection(2, "favorites")]; + const files = [file(1, 1, 0), file(2, 1, 1), file(10, 2, 0)]; + const cache = new FakeCache(); + const pre = new Precache({ originalsDays: 7 }); + pre.bind(cache); + pre.update(records(cols, files)); + + const before = pre.status(); + expect(before.thumbnailsTotal).toBe(3); + expect(before.thumbnailsCached).toBe(0); + expect(before.originalsPinned).toBe(3); // files 1, 2, 10 all in window + expect(before.originalsCached).toBe(0); + + pre.start(); + await until( + () => + cache.thumbFetched.length === 3 && + cache.originalFetched.length === 3, + ); + const after = pre.status(); + expect(after.thumbnailsCached).toBe(3); + expect(after.originalsCached).toBe(3); + }); + + it("honours the disable flags", async () => { + const cols = [collection(1)]; + const files = [file(1, 1, 0)]; + const cache = new FakeCache(); + const pre = new Precache({ thumbnails: false, originals: false }); + pre.bind(cache); + pre.update(records(cols, files)); + pre.start(); + + await new Promise((r) => setTimeout(r, 20)); + expect(cache.thumbFetched).toEqual([]); + expect(cache.originalFetched).toEqual([]); + expect(pre.isPinned(1)).toBe(false); + expect(pre.status().thumbnailsTotal).toBe(0); + }); +}); + +// ---- Integration through the real ContentCache and Library ---- + +const enteFile = (id: number, collectionID: number): EnteFile => + file(id, collectionID, 0); + +describe("Precache eviction integration", () => { + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-precache-evict-")); + }); + afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); + }); + + it("never evicts a pinned original the precache put in place", async () => { + const cacheDir = join(root, "cache"); + // File 1 is the favorites album's only file (pinned regardless of age); + // files 2 and 3 sit outside the latest-week window, so only file 1 is + // pinned. The by-id map serves the bytes for each fetch. + const cols = [collection(1), collection(2, "favorites")]; + const files = [file(1, 2, 30), file(2, 1, 40), file(3, 1, 50)]; + const byID = new Map(files.map((f) => [f.id, f])); + const source: ContentSource = { + original: async ({ destination }) => { + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + thumbnail: async ({ destination }) => { + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + }; + const statfs: StatFsFn = async () => ({ + bsize: 1, + bavail: 1_000_000_000, + }); + const pre = new Precache({ originalsDays: 7 }); + const cache = new ContentCache({ + pools: new RequestPools(), + source, + cacheDirectory: cacheDir, + getFile: (id) => byID.get(id), + statfs, + cacheOriginalsMaxBytes: 25, // holds two 10-byte originals + freeBelowBytes: 0, + isPinned: (id) => pre.isPinned(id), + }); + pre.bind(cache); + pre.update(records(cols, files)); + expect(pre.isPinned(1)).toBe(true); + expect(pre.isPinned(2)).toBe(false); + await cache.open(); + + // Fill three originals; the 25-byte cap forces an eviction on the + // third, and the pinned file 1 must survive it even though it is the + // least-recently-used. + await cache.original(1); + utimesSync(join(cacheDir, "originals", "1.jpg"), 1000, 1000); // oldest + await cache.original(2); + utimesSync(join(cacheDir, "originals", "2.jpg"), 2000, 2000); + await cache.original(3); + + expect(existsSync(join(cacheDir, "originals", "1.jpg"))).toBe(true); + expect(existsSync(join(cacheDir, "originals", "2.jpg"))).toBe(false); + expect(existsSync(join(cacheDir, "originals", "3.jpg"))).toBe(true); + }); +}); + +describe("Precache preemption", () => { + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-precache-preempt-")); + }); + afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); + }); + + it("lets an on-demand original preempt the background originals fill", async () => { + const byID = new Map([ + [1, enteFile(1, 1)], + [2, enteFile(2, 1)], + [3, enteFile(3, 1)], + ]); + const finished: number[] = []; + let openGate!: () => void; + const gate = new Promise((r) => (openGate = r)); + let sawFirst!: () => void; + const firstStarted = new Promise((r) => (sawFirst = r)); + let started = 0; + const source: ContentSource = { + original: async ({ file: f, destination }) => { + if (++started === 1) sawFirst(); + await gate; + await writeFile(destination, Buffer.alloc(10, 1)); + finished.push(f.id); + return { bytesWritten: 10 }; + }, + thumbnail: async ({ destination }) => { + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + }; + // One content slot, so file 1 holds it while 2 and 3 wait. + const cache = new ContentCache({ + pools: new RequestPools({ contentConcurrency: 1 }), + source, + cacheDirectory: join(root, "cache"), + getFile: (id) => byID.get(id), + statfs: async () => ({ bsize: 1, bavail: 1_000_000_000 }), + freeBelowBytes: 0, + }); + await cache.open(); + + const pA = cache.ensureOriginals({ fileIDs: [1] }); // background + await firstStarted; // file 1 now holds the only slot + const pB = cache.original(2); // on-demand, queued behind file 1 + const pC = cache.ensureOriginals({ fileIDs: [3] }); // background, queued + await new Promise((r) => setTimeout(r, 5)); // let both enqueue + openGate(); + await Promise.all([pA, pB, pC]); + + // On-demand file 2 was served before the background file 3. + expect(finished).toEqual([1, 2, 3]); + }); +}); + +describe("Precache through Library.open", () => { + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-precache-lib-")); + }); + afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); + }); + + class MockClient { + served = false; + whoami(): { email: string; userID: number } { + return { email: "u@example.com", userID: 7 }; + } + async collectionsSince(): Promise { + if (this.served) return { collections: [], deleted: [], cursor: 1 }; + this.served = true; + return { + collections: [collection(1), collection(2, "favorites")], + deleted: [], + cursor: 1, + }; + } + async filesSince(args: { collectionID: number }): Promise { + const files = + args.collectionID === 1 + ? [enteFile(1, 1), enteFile(2, 1)] + : [enteFile(3, 2)]; + return { files, deleted: [], cursor: 1 }; + } + } + + it("starts both precaches from open() and reports them in status()", async () => { + const thumbFetched = new Set(); + const origFetched = new Set(); + const source: ContentSource = { + original: async ({ file: f, destination }) => { + origFetched.add(f.id); + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + thumbnail: async ({ file: f, destination }) => { + thumbFetched.add(f.id); + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + }; + const lib = await Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + contentSource: source, + refreshIntervalSeconds: 3600, + }); + + // Every file's thumbnail is precached; the favorite (file 3) and the + // week's files (1, 2) all have their originals precached. + await until(() => thumbFetched.size === 3 && origFetched.size === 3); + const status = lib.status(); + expect(status.thumbnailsTotal).toBe(3); + expect(status.thumbnailsCached).toBe(3); + expect(status.originalsPinned).toBe(3); + expect(status.originalsCached).toBe(3); + lib.close(); + }); +});