diff --git a/src/library/content.ts b/src/library/content.ts index a5379ec..8828130 100644 --- a/src/library/content.ts +++ b/src/library/content.ts @@ -23,8 +23,16 @@ // decrypted length this layer has. import { existsSync, statSync } from "node:fs"; -import { chmod, mkdir, readdir, rm, stat } from "node:fs/promises"; -import { extname, join } from "node:path"; +import { + chmod, + mkdir, + readdir, + rm, + stat, + statfs, + utimes, +} from "node:fs/promises"; +import { dirname, extname, join } from "node:path"; import type { ApiClient } from "../api/client.js"; import { @@ -39,6 +47,11 @@ const DIR_MODE = 0o700; const FILE_MODE = 0o600; const TEMP_PREFIX = ".quak-"; const TEMP_SUFFIX = ".tmp"; +const GIB = 1024 * 1024 * 1024; +// Owner ruling (#36): bound the originals cache at 100 GiB, but back off when +// the volume has under 50 GiB free so the cache never crowds the disk. +export const DEFAULT_ORIGINALS_MAX_BYTES = 100 * GIB; +export const DEFAULT_FREE_BELOW_BYTES = 50 * GIB; // Ente thumbnails are always JPEG, so the cache stores them with a fixed // extension rather than deriving one from the (image or video) title. const THUMBNAIL_EXT = ".jpg"; @@ -133,6 +146,28 @@ export interface CachedPaths { thumbnailPath?: string; } +// The slice of `fs.statfs` the eviction limit needs: `bavail` is the blocks +// available to an unprivileged writer and `bsize` their size, so +// `bavail * bsize` is the free byte count. Injectable so tests drive the +// adaptive limit without a real volume. +export interface StatFsResult { + bsize: number; + bavail: number; +} +export type StatFsFn = (path: string) => Promise; + +const realStatFs: StatFsFn = async (path) => { + const s = await statfs(path); + return { bsize: s.bsize, bavail: s.bavail }; +}; + +// The current usage and effective limit of the originals cache, in bytes. +// `limitBytes` is the adaptive ceiling last computed (see `originalsLimit`). +export interface OriginalsStatus { + usedBytes: number; + limitBytes?: number; +} + export interface ContentCacheOptions { pools: RequestPools; source: ContentSource; @@ -144,6 +179,18 @@ export interface ContentCacheOptions { // Resolve any membership of a file; every membership shares the underlying // content key, so any one decrypts the same bytes. getFile: (fileID: number) => EnteFile | undefined; + // Hard ceiling on `cacheDirectory/originals` (default 100 GiB) and the free + // space to protect on the volume (default 50 GiB). The effective limit is + // the lesser of the ceiling and what fits above the protected free space. + cacheOriginalsMaxBytes?: number; + freeBelowBytes?: number; + // Whether an original is pinned (favorites + latest week; the precache unit + // #48 supplies the set). Pinned originals are never evicted; when only + // pinned originals remain the cache runs over-limit until the set shrinks. + isPinned?: (fileID: number) => boolean; + // Free-space probe on the volume holding `cacheDirectory`; defaults to the + // real `fs.statfs`. + statfs?: StatFsFn; } // Thrown inside a pooled task to drop a queued fetch that was aborted before it @@ -190,6 +237,17 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { // listing at open() and extended as fetches store new files. private readonly originals = new Map(); private readonly thumbnails = new Map(); + private readonly maxOriginalsBytes: number; + private readonly freeBelowBytes: number; + private readonly isPinned: (fileID: number) => boolean; + private readonly statfs: StatFsFn; + // The last measured usage and effective limit, refreshed at open() and after + // every original write; exposed through `originalsStatus`. + private originalsUsedBytes = 0; + private originalsLimitBytes?: number; + // Serializes limit enforcement so concurrent original writes never race on + // the map or delete each other's just-freed room. + private enforcing: Promise = Promise.resolve(); constructor(opts: ContentCacheOptions) { this.pools = opts.pools; @@ -198,6 +256,11 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { this.getFile = opts.getFile; this.originalsDir = join(opts.cacheDirectory, "originals"); this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails"); + this.maxOriginalsBytes = + opts.cacheOriginalsMaxBytes ?? DEFAULT_ORIGINALS_MAX_BYTES; + this.freeBelowBytes = opts.freeBelowBytes ?? DEFAULT_FREE_BELOW_BYTES; + this.isPinned = opts.isPinned ?? (() => false); + this.statfs = opts.statfs ?? realStatFs; } // Prepare the cache directories, reap orphan temp files, and take the @@ -207,6 +270,18 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { await this.ensureDir(this.thumbnailsDir); await this.scan(this.originalsDir, this.originals); await this.scan(this.thumbnailsDir, this.thumbnails); + // Publish the current usage and limit without evicting; a restart + // reuses whatever survived on disk. Eviction only ever fires on a write. + await this.refreshOriginalsLimit(); + } + + // The current originals usage and effective limit, both in bytes, as of the + // last write or open. `status().originalsLimitBytes` surfaces this. + originalsStatus(): OriginalsStatus { + return { + usedBytes: this.originalsUsedBytes, + limitBytes: this.originalsLimitBytes, + }; } // The cache paths known for a file, for the record projection to expose as @@ -321,8 +396,16 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { const cached = known.get(fileID); if (cached !== undefined) { const size = fileSize(cached); - if (size !== undefined && size > 0) + if (size !== undefined && size > 0) { + // Returning an original's path is a use: bump its mtime so LRU + // order reflects it and survives a restart with no ledger. + if ( + kind === "original" && + dirname(cached) === this.originalsDir + ) + await this.touch(cached); return { path: cached, bytes: size, cached: true }; + } // A recorded file that has since gone re-fetches below. known.delete(fileID); } @@ -367,6 +450,10 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { ); } known.set(fileID, dest); + // A fresh original may have crossed the limit; make room by + // evicting least-recently-used originals (never this one, just + // touched). Thumbnails are never bounded or evicted. + if (kind === "original") await this.enforceOriginalsLimit(); return { path: dest, bytes: size, cached: false }; }, { priority, key: fileID }, @@ -387,6 +474,103 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { return result.bytesWritten; } + // Best-effort bump of a file's mtime to now; a failed touch must never fail + // the read it accompanies. + private async touch(path: string): Promise { + const now = new Date(); + await utimes(path, now, now).catch(() => undefined); + } + + // Every stored original that lives under `originalsDir` (a backup-directory + // hit recorded in the map is excluded), with its size and mtime. Entries + // whose file has vanished are dropped from the map. Backups and thumbnails + // are never counted. + private async measureOriginals(): Promise<{ + entries: { + fileID: number; + path: string; + size: number; + mtimeMs: number; + }[]; + used: number; + }> { + const entries: { + fileID: number; + path: string; + size: number; + mtimeMs: number; + }[] = []; + let used = 0; + for (const [fileID, path] of this.originals) { + if (dirname(path) !== this.originalsDir) continue; + try { + const s = await stat(path); + entries.push({ + fileID, + path, + size: s.size, + mtimeMs: s.mtimeMs, + }); + used += s.size; + } catch { + this.originals.delete(fileID); + } + } + return { entries, used }; + } + + // The effective ceiling on originals: the configured max, but no more than + // what fits once the protected free space is set aside. `used + free` is the + // volume space the cache could occupy; subtracting `freeBelowBytes` leaves + // the reserve untouched. Clamped at zero. + private async originalsLimit(used: number): Promise { + const { bsize, bavail } = await this.statfs(this.originalsDir); + const free = bsize * bavail; + const adaptive = used + free - this.freeBelowBytes; + return Math.max(0, Math.min(this.maxOriginalsBytes, adaptive)); + } + + // Recompute and publish usage and limit without evicting (used at open()). + private refreshOriginalsLimit(): Promise { + return this.serializeEnforce(async () => { + const { used } = await this.measureOriginals(); + this.originalsUsedBytes = used; + this.originalsLimitBytes = await this.originalsLimit(used); + }); + } + + // Evict least-recently-used originals until usage fits the limit. Pinned + // originals are skipped; when only pinned originals remain the cache stays + // over-limit until the pinned set shrinks. + private enforceOriginalsLimit(): Promise { + return this.serializeEnforce(async () => { + const { entries, used } = await this.measureOriginals(); + const limit = await this.originalsLimit(used); + let remaining = used; + if (remaining > limit) { + const evictable = entries + .filter((e) => !this.isPinned(e.fileID)) + .sort((a, b) => a.mtimeMs - b.mtimeMs); + for (const e of evictable) { + if (remaining <= limit) break; + await rm(e.path, { force: true }); + this.originals.delete(e.fileID); + remaining -= e.size; + } + } + this.originalsUsedBytes = remaining; + this.originalsLimitBytes = limit; + }); + } + + // Run limit work one at a time; failures are swallowed so a transient + // statfs or unlink error never rejects the read or write that triggered it. + private serializeEnforce(work: () => Promise): Promise { + const next = this.enforcing.then(work).catch(() => undefined); + this.enforcing = next; + return next; + } + private async ensureDir(dir: string): Promise { // chmod after mkdir so the mode is tightened even when the directory // already existed with a looser one; mkdir alone would not. diff --git a/src/library/index.ts b/src/library/index.ts index 04e3fcd..c55caad 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -132,6 +132,14 @@ export interface LibraryOptions { // Overrides the client's own `contentSource()`; mainly for tests that drive // the cache with a stand-in source. contentSource?: ContentSource; + // Bound on `cacheDirectory/originals` (default 100 GiB) and the free space + // to protect on its volume (default 50 GiB). The effective limit adapts + // down as the disk fills; `status().originalsLimitBytes` reports it. + cacheOriginalsMaxBytes?: number; + freeBelowBytes?: number; + // Whether an original is pinned and so never evicted (favorites + latest + // week; the precache unit #48 supplies the set). + isOriginalPinned?: (fileID: number) => boolean; } export interface LibraryStatus { @@ -153,6 +161,10 @@ export interface LibraryStatus { // when ML fetching is disabled. mlStored?: number; mlIndexed?: number; + // Bytes stored in the originals cache and the effective size limit as of the + // last write or open; undefined when no content cache is open. + originalsUsedBytes?: number; + originalsLimitBytes?: number; closed: boolean; } @@ -289,6 +301,9 @@ export class Library { cacheDirectory, downloadDirectory: opts.downloadDirectory, getFile: (fileID) => store.getFileByID(fileID), + cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes, + freeBelowBytes: opts.freeBelowBytes, + isPinned: opts.isOriginalPinned, }); await cache.open(); } @@ -364,6 +379,7 @@ export class Library { files += this.store.listFiles(c.id).length; } const ml = this.mldata?.stats(); + const originals = this.cache?.originalsStatus(); return { userID: this.store.userID, collections: collections.length, @@ -374,6 +390,8 @@ export class Library { lastMLError: this.lastMLError, mlStored: ml?.stored, mlIndexed: ml?.indexed, + originalsUsedBytes: originals?.usedBytes, + originalsLimitBytes: originals?.limitBytes, closed: this.closed, }; } diff --git a/test/library/content-eviction.test.ts b/test/library/content-eviction.test.ts new file mode 100644 index 0000000..bc26420 --- /dev/null +++ b/test/library/content-eviction.test.ts @@ -0,0 +1,268 @@ +/** + * Tests for the originals cache size limit and LRU eviction (issue #47), + * layered on the on-disk content cache (#46). + * + * Only `cacheDirectory/originals` is bounded and evicted. Before each original + * write the effective limit is + * min(cacheOriginalsMaxBytes, bytesUsedByOriginals + bytesFree - freeBelowBytes) + * with `bytesFree` read from `statfs` on the volume holding `cacheDirectory`. + * The limit therefore falls as the disk fills and rises as space returns. When + * a write would cross the limit, least-recently-used originals are removed until + * it fits; pinned files are skipped, and if only pinned files remain the write + * proceeds over-limit. Last-use is the file `mtime`, bumped whenever a read + * returns an original's path, so ordering survives a restart with no ledger. + * + * `statfs` is injected so the adaptive limit is exercised deterministically: + * `bsize` is 1, so `bavail` is the free byte count the formula sees. The source + * writes a controllable number of bytes per file, and tests set each stored + * file's `mtime` explicitly so LRU order does not depend on wall-clock timing. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, existsSync, statSync, utimesSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + ContentCache, + type ContentSource, + type StatFsFn, +} from "../../src/library/content.js"; +import { RequestPools } from "../../src/library/pools.js"; +import type { EnteFile } from "../../src/model/types.js"; + +const file = (id: number): EnteFile => ({ + id, + collectionID: 1, + ownerID: 1, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: 0, + modificationTime: 0, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 0, +}); + +// A source that writes a controllable number of bytes per file (default 10). +class SizedSource implements ContentSource { + sizeFor = new Map(); + + async original(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + const n = this.sizeFor.get(args.file.id) ?? 10; + await writeFile(args.destination, Buffer.alloc(n, 1)); + return { bytesWritten: n }; + } + + async thumbnail(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + await writeFile(args.destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + } +} + +let root: string; +let cacheDir: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-eviction-")); + cacheDir = join(root, "cache"); +}); + +afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); +}); + +const originalPath = (id: number): string => + join(cacheDir, "originals", `${id}.jpg`); + +// Pin an original's mtime to a fixed second-resolution instant so LRU order is +// deterministic. Lower `seconds` = older = evicted first. +const setMtime = (id: number, seconds: number): void => { + utimesSync(originalPath(id), seconds, seconds); +}; + +const buildCache = (args: { + source?: ContentSource; + files?: EnteFile[]; + statfs: StatFsFn; + cacheOriginalsMaxBytes?: number; + freeBelowBytes?: number; + isPinned?: (fileID: number) => boolean; +}): { cache: ContentCache; source: SizedSource } => { + const source = (args.source as SizedSource) ?? new SizedSource(); + const byID = new Map(); + for (const f of args.files ?? [file(1), file(2), file(3), file(4), file(5)]) + byID.set(f.id, f); + const cache = new ContentCache({ + pools: new RequestPools(), + source, + cacheDirectory: cacheDir, + getFile: (id) => byID.get(id), + statfs: args.statfs, + cacheOriginalsMaxBytes: args.cacheOriginalsMaxBytes, + freeBelowBytes: args.freeBelowBytes, + isPinned: args.isPinned, + }); + return { cache, source }; +}; + +// Plenty of free space, so the configured max governs the limit unless a test +// dials it down. `bsize` of 1 makes `bavail` the free byte count. +const abundantFree: StatFsFn = async () => ({ + bsize: 1, + bavail: 1_000_000_000, +}); + +describe("originals eviction", () => { + it("removes least-recently-used originals to fit the limit", async () => { + // Each original is 10 bytes; a 25-byte cap holds two. + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 25, + freeBelowBytes: 0, + }); + await cache.open(); + + await cache.original(1); + setMtime(1, 1000); + await cache.original(2); + setMtime(2, 2000); + // Writing the third (total 30 > 25) evicts the oldest, file 1. + await cache.original(3); + + expect(existsSync(originalPath(1))).toBe(false); + expect(existsSync(originalPath(2))).toBe(true); + expect(existsSync(originalPath(3))).toBe(true); + expect(cache.pathsFor(1).originalPath).toBeUndefined(); + expect(cache.originalsStatus().usedBytes).toBe(20); + }); + + it("skips pinned files, evicting the oldest unpinned instead", async () => { + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 25, + freeBelowBytes: 0, + isPinned: (id) => id === 1, + }); + await cache.open(); + + await cache.original(1); + setMtime(1, 1000); // oldest, but pinned + await cache.original(2); + setMtime(2, 2000); + await cache.original(3); + + // File 1 is oldest but pinned, so file 2 is evicted instead. + expect(existsSync(originalPath(1))).toBe(true); + expect(existsSync(originalPath(2))).toBe(false); + expect(existsSync(originalPath(3))).toBe(true); + }); + + it("proceeds over-limit when only pinned files remain", async () => { + // A 5-byte cap cannot hold even one 10-byte original. + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 5, + freeBelowBytes: 0, + isPinned: () => true, + }); + await cache.open(); + + const result = await cache.original(1); + + expect(existsSync(result.path)).toBe(true); + const status = cache.originalsStatus(); + expect(status.usedBytes).toBe(10); + expect(status.usedBytes).toBeGreaterThan(status.limitBytes ?? 0); + }); + + it("adapts the limit down as the disk fills and up as space returns", async () => { + // The configured max is generous; free space drives the limit. Eviction + // only kicks in once free space falls below the protected reserve. + let free = 1_000_000; + const statfs: StatFsFn = async () => ({ bsize: 1, bavail: free }); + const { cache } = buildCache({ + statfs, + cacheOriginalsMaxBytes: 1000, + freeBelowBytes: 100, + }); + await cache.open(); + + // Ample free space: the limit is the configured max and nothing is + // evicted as three 10-byte originals accumulate. + await cache.original(1); + setMtime(1, 1000); + await cache.original(2); + setMtime(2, 2000); + await cache.original(3); + setMtime(3, 3000); + expect(cache.originalsStatus().limitBytes).toBe(1000); + expect(cache.originalsStatus().usedBytes).toBe(30); + + // The disk fills: only 95 bytes free, below the 100-byte reserve. The + // next write (used 40) sees limit = min(1000, 40 + 95 - 100) = 35 and + // evicts the oldest original (file 1) to fit. + free = 95; + await cache.original(4); + expect(cache.originalsStatus().limitBytes).toBe(35); + expect(existsSync(originalPath(1))).toBe(false); + expect(cache.originalsStatus().usedBytes).toBe(30); + setMtime(4, 4000); + + // Space returns: the limit rises back to the configured max and the + // next write is kept without eviction. + free = 1_000_000; + await cache.original(5); + expect(cache.originalsStatus().limitBytes).toBe(1000); + expect(existsSync(originalPath(4))).toBe(true); + expect(existsSync(originalPath(5))).toBe(true); + }); + + it("bumps an original's mtime when a read returns its path", async () => { + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 1000, + freeBelowBytes: 0, + }); + await cache.open(); + + await cache.original(1); + // Age the file well into the past. + setMtime(1, 1000); + expect(statSync(originalPath(1)).mtimeMs).toBeLessThan(2_000_000); + + // A second read is a cache hit that must touch the file. + const before = Date.now(); + await cache.original(1); + expect(statSync(originalPath(1)).mtimeMs).toBeGreaterThanOrEqual( + before - 2000, + ); + }); + + it("never counts or evicts thumbnails", async () => { + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 5, + freeBelowBytes: 0, + }); + await cache.open(); + + await cache.thumbnail(1); + await cache.thumbnail(2); + + expect(cache.pathsFor(1).thumbnailPath).toBeDefined(); + expect(cache.pathsFor(2).thumbnailPath).toBeDefined(); + expect(cache.originalsStatus().usedBytes).toBe(0); + }); +});