Bound the originals cache with adaptive LRU eviction (closes #47)

Bounds cacheDirectory/originals to an adaptive limit:
min(cacheOriginalsMaxBytes, bytesUsed + bytesFree - freeBelowBytes), with
bytesFree from fs.statfs, so it falls as the disk fills and rises as space
returns (status().originalsLimitBytes exposes it). Each original write evicts
least-recently-used originals until usage fits; last-use is the file mtime,
bumped on every read that returns a path, so order survives restarts. Pinned
originals and the file just written are never evicted — so an on-demand fetch
larger than the limit keeps the path it returns and re-downloads nothing; the
skipped file becomes eligible on a later write. When only such files remain the
cache stays over-limit. Defaults: 100 GiB limit, 50 GiB reserve; thumbnails and
the backup directory are never counted.

Model: opus-4-8

src/library/content.ts

@@ -23,8 +23,16 @@
 // decrypted length this layer has.
 
 import { existsSync, statSync } from "node:fs";
-import { chmod, mkdir, readdir, rm, stat } from "node:fs/promises";
-import { extname, join } from "node:path";
+import {
+    chmod,
+    mkdir,
+    readdir,
+    rm,
+    stat,
+    statfs,
+    utimes,
+} from "node:fs/promises";
+import { dirname, extname, join } from "node:path";
 
 import type { ApiClient } from "../api/client.js";
 import {
@@ -39,6 +47,11 @@ const DIR_MODE = 0o700;
 const FILE_MODE = 0o600;
 const TEMP_PREFIX = ".quak-";
 const TEMP_SUFFIX = ".tmp";
+const GIB = 1024 * 1024 * 1024;
+// Owner ruling (#36): bound the originals cache at 100 GiB, but back off when
+// the volume has under 50 GiB free so the cache never crowds the disk.
+export const DEFAULT_ORIGINALS_MAX_BYTES = 100 * GIB;
+export const DEFAULT_FREE_BELOW_BYTES = 50 * GIB;
 // Ente thumbnails are always JPEG, so the cache stores them with a fixed
 // extension rather than deriving one from the (image or video) title.
 const THUMBNAIL_EXT = ".jpg";
@@ -133,6 +146,28 @@ export interface CachedPaths {
     thumbnailPath?: string;
 }
 
+// The slice of `fs.statfs` the eviction limit needs: `bavail` is the blocks
+// available to an unprivileged writer and `bsize` their size, so
+// `bavail * bsize` is the free byte count. Injectable so tests drive the
+// adaptive limit without a real volume.
+export interface StatFsResult {
+    bsize: number;
+    bavail: number;
+}
+export type StatFsFn = (path: string) => Promise<StatFsResult>;
+
+const realStatFs: StatFsFn = async (path) => {
+    const s = await statfs(path);
+    return { bsize: s.bsize, bavail: s.bavail };
+};
+
+// The current usage and effective limit of the originals cache, in bytes.
+// `limitBytes` is the adaptive ceiling last computed (see `originalsLimit`).
+export interface OriginalsStatus {
+    usedBytes: number;
+    limitBytes?: number;
+}
+
 export interface ContentCacheOptions {
     pools: RequestPools;
     source: ContentSource;
@@ -144,6 +179,18 @@ export interface ContentCacheOptions {
     // Resolve any membership of a file; every membership shares the underlying
     // content key, so any one decrypts the same bytes.
     getFile: (fileID: number) => EnteFile | undefined;
+    // Hard ceiling on `cacheDirectory/originals` (default 100 GiB) and the free
+    // space to protect on the volume (default 50 GiB). The effective limit is
+    // the lesser of the ceiling and what fits above the protected free space.
+    cacheOriginalsMaxBytes?: number;
+    freeBelowBytes?: number;
+    // Whether an original is pinned (favorites + latest week; the precache unit
+    // #48 supplies the set). Pinned originals are never evicted; when only
+    // pinned originals remain the cache runs over-limit until the set shrinks.
+    isPinned?: (fileID: number) => boolean;
+    // Free-space probe on the volume holding `cacheDirectory`; defaults to the
+    // real `fs.statfs`.
+    statfs?: StatFsFn;
 }
 
 // Thrown inside a pooled task to drop a queued fetch that was aborted before it
@@ -190,6 +237,17 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
     // listing at open() and extended as fetches store new files.
     private readonly originals = new Map<number, string>();
     private readonly thumbnails = new Map<number, string>();
+    private readonly maxOriginalsBytes: number;
+    private readonly freeBelowBytes: number;
+    private readonly isPinned: (fileID: number) => boolean;
+    private readonly statfs: StatFsFn;
+    // The last measured usage and effective limit, refreshed at open() and after
+    // every original write; exposed through `originalsStatus`.
+    private originalsUsedBytes = 0;
+    private originalsLimitBytes?: number;
+    // Serializes limit enforcement so concurrent original writes never race on
+    // the map or delete each other's just-freed room.
+    private enforcing: Promise<void> = Promise.resolve();
 
     constructor(opts: ContentCacheOptions) {
         this.pools = opts.pools;
@@ -198,6 +256,11 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
         this.getFile = opts.getFile;
         this.originalsDir = join(opts.cacheDirectory, "originals");
         this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails");
+        this.maxOriginalsBytes =
+            opts.cacheOriginalsMaxBytes ?? DEFAULT_ORIGINALS_MAX_BYTES;
+        this.freeBelowBytes = opts.freeBelowBytes ?? DEFAULT_FREE_BELOW_BYTES;
+        this.isPinned = opts.isPinned ?? (() => false);
+        this.statfs = opts.statfs ?? realStatFs;
     }
 
     // Prepare the cache directories, reap orphan temp files, and take the
@@ -207,6 +270,18 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
         await this.ensureDir(this.thumbnailsDir);
         await this.scan(this.originalsDir, this.originals);
         await this.scan(this.thumbnailsDir, this.thumbnails);
+        // Publish the current usage and limit without evicting; a restart
+        // reuses whatever survived on disk. Eviction only ever fires on a write.
+        await this.refreshOriginalsLimit();
+    }
+
+    // The current originals usage and effective limit, both in bytes, as of the
+    // last write or open. `status().originalsLimitBytes` surfaces this.
+    originalsStatus(): OriginalsStatus {
+        return {
+            usedBytes: this.originalsUsedBytes,
+            limitBytes: this.originalsLimitBytes,
+        };
     }
 
     // The cache paths known for a file, for the record projection to expose as
@@ -321,8 +396,16 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
         const cached = known.get(fileID);
         if (cached !== undefined) {
             const size = fileSize(cached);
-            if (size !== undefined && size > 0)
+            if (size !== undefined && size > 0) {
+                // Returning an original's path is a use: bump its mtime so LRU
+                // order reflects it and survives a restart with no ledger.
+                if (
+                    kind === "original" &&
+                    dirname(cached) === this.originalsDir
+                )
+                    await this.touch(cached);
                 return { path: cached, bytes: size, cached: true };
+            }
             // A recorded file that has since gone re-fetches below.
             known.delete(fileID);
         }
@@ -367,6 +450,13 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
                     );
                 }
                 known.set(fileID, dest);
+                // A fresh original may have crossed the limit; make room by
+                // evicting least-recently-used originals. The file just written
+                // is excluded from the candidates so an on-demand fetch that is
+                // itself over budget proceeds over-limit and returns a path that
+                // exists on disk. Thumbnails are never bounded or evicted.
+                if (kind === "original")
+                    await this.enforceOriginalsLimit(fileID);
                 return { path: dest, bytes: size, cached: false };
             },
             { priority, key: fileID },
@@ -387,6 +477,110 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
         return result.bytesWritten;
     }
 
+    // Best-effort bump of a file's mtime to now; a failed touch must never fail
+    // the read it accompanies.
+    private async touch(path: string): Promise<void> {
+        const now = new Date();
+        await utimes(path, now, now).catch(() => undefined);
+    }
+
+    // Every stored original that lives under `originalsDir` (a backup-directory
+    // hit recorded in the map is excluded), with its size and mtime. Entries
+    // whose file has vanished are dropped from the map. Backups and thumbnails
+    // are never counted.
+    private async measureOriginals(): Promise<{
+        entries: {
+            fileID: number;
+            path: string;
+            size: number;
+            mtimeMs: number;
+        }[];
+        used: number;
+    }> {
+        const entries: {
+            fileID: number;
+            path: string;
+            size: number;
+            mtimeMs: number;
+        }[] = [];
+        let used = 0;
+        for (const [fileID, path] of this.originals) {
+            if (dirname(path) !== this.originalsDir) continue;
+            try {
+                const s = await stat(path);
+                entries.push({
+                    fileID,
+                    path,
+                    size: s.size,
+                    mtimeMs: s.mtimeMs,
+                });
+                used += s.size;
+            } catch {
+                this.originals.delete(fileID);
+            }
+        }
+        return { entries, used };
+    }
+
+    // The effective ceiling on originals: the configured max, but no more than
+    // what fits once the protected free space is set aside. `used + free` is the
+    // volume space the cache could occupy; subtracting `freeBelowBytes` leaves
+    // the reserve untouched. Clamped at zero.
+    private async originalsLimit(used: number): Promise<number> {
+        const { bsize, bavail } = await this.statfs(this.originalsDir);
+        const free = bsize * bavail;
+        const adaptive = used + free - this.freeBelowBytes;
+        return Math.max(0, Math.min(this.maxOriginalsBytes, adaptive));
+    }
+
+    // Recompute and publish usage and limit without evicting (used at open()).
+    private refreshOriginalsLimit(): Promise<void> {
+        return this.serializeEnforce(async () => {
+            const { used } = await this.measureOriginals();
+            this.originalsUsedBytes = used;
+            this.originalsLimitBytes = await this.originalsLimit(used);
+        });
+    }
+
+    // Evict least-recently-used originals until usage fits the limit. Pinned
+    // originals are skipped, as is `justWrittenID` (the original this write just
+    // stored, so an over-budget on-demand fetch never evicts the file it is
+    // about to return). When only such originals remain the cache stays
+    // over-limit until the protected set shrinks; the skipped file becomes
+    // eligible on a later write.
+    private enforceOriginalsLimit(justWrittenID: number): Promise<void> {
+        return this.serializeEnforce(async () => {
+            const { entries, used } = await this.measureOriginals();
+            const limit = await this.originalsLimit(used);
+            let remaining = used;
+            if (remaining > limit) {
+                const evictable = entries
+                    .filter(
+                        (e) =>
+                            e.fileID !== justWrittenID &&
+                            !this.isPinned(e.fileID),
+                    )
+                    .sort((a, b) => a.mtimeMs - b.mtimeMs);
+                for (const e of evictable) {
+                    if (remaining <= limit) break;
+                    await rm(e.path, { force: true });
+                    this.originals.delete(e.fileID);
+                    remaining -= e.size;
+                }
+            }
+            this.originalsUsedBytes = remaining;
+            this.originalsLimitBytes = limit;
+        });
+    }
+
+    // Run limit work one at a time; failures are swallowed so a transient
+    // statfs or unlink error never rejects the read or write that triggered it.
+    private serializeEnforce(work: () => Promise<void>): Promise<void> {
+        const next = this.enforcing.then(work).catch(() => undefined);
+        this.enforcing = next;
+        return next;
+    }
+
     private async ensureDir(dir: string): Promise<void> {
         // chmod after mkdir so the mode is tightened even when the directory
         // already existed with a looser one; mkdir alone would not.

src/library/index.ts

@@ -132,6 +132,14 @@ export interface LibraryOptions {
     // Overrides the client's own `contentSource()`; mainly for tests that drive
     // the cache with a stand-in source.
     contentSource?: ContentSource;
+    // Bound on `cacheDirectory/originals` (default 100 GiB) and the free space
+    // to protect on its volume (default 50 GiB). The effective limit adapts
+    // down as the disk fills; `status().originalsLimitBytes` reports it.
+    cacheOriginalsMaxBytes?: number;
+    freeBelowBytes?: number;
+    // Whether an original is pinned and so never evicted (favorites + latest
+    // week; the precache unit #48 supplies the set).
+    isOriginalPinned?: (fileID: number) => boolean;
 }
 
 export interface LibraryStatus {
@@ -153,6 +161,10 @@ export interface LibraryStatus {
     // when ML fetching is disabled.
     mlStored?: number;
     mlIndexed?: number;
+    // Bytes stored in the originals cache and the effective size limit as of the
+    // last write or open; undefined when no content cache is open.
+    originalsUsedBytes?: number;
+    originalsLimitBytes?: number;
     closed: boolean;
 }
 
@@ -289,6 +301,9 @@ export class Library {
                 cacheDirectory,
                 downloadDirectory: opts.downloadDirectory,
                 getFile: (fileID) => store.getFileByID(fileID),
+                cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes,
+                freeBelowBytes: opts.freeBelowBytes,
+                isPinned: opts.isOriginalPinned,
             });
             await cache.open();
         }
@@ -364,6 +379,7 @@ export class Library {
             files += this.store.listFiles(c.id).length;
         }
         const ml = this.mldata?.stats();
+        const originals = this.cache?.originalsStatus();
         return {
             userID: this.store.userID,
             collections: collections.length,
@@ -374,6 +390,8 @@ export class Library {
             lastMLError: this.lastMLError,
             mlStored: ml?.stored,
             mlIndexed: ml?.indexed,
+            originalsUsedBytes: originals?.usedBytes,
+            originalsLimitBytes: originals?.limitBytes,
             closed: this.closed,
         };
     }