Bound the originals cache with adaptive LRU eviction (closes #47)
check / check (push) Successful in 35s
check / check (push) Successful in 35s
Bounds cacheDirectory/originals to an adaptive limit: min(configured max, bytesUsed + bytesFree - reserve), bytesFree from statfs, so it tracks disk pressure (status().originalsLimitBytes exposes it). Each original write evicts least-recently-used originals until usage fits; last-use is the file mtime, bumped on every read that returns a path, so order survives restarts. Never evicted: pinned originals and every original whose write overlaps the evicting one — its own and any concurrent sibling's, since content writes run in parallel. So an over-budget fetch keeps the path it returns, and no write can delete a sibling's file before that fetch returns it; such files become eligible on a later, non-overlapping write. Defaults: 100 GiB limit, 50 GiB reserve. Model: opus-4-8
This commit is contained in:
+234
-3
@@ -23,8 +23,16 @@
|
||||
// decrypted length this layer has.
|
||||
|
||||
import { existsSync, statSync } from "node:fs";
|
||||
import { chmod, mkdir, readdir, rm, stat } from "node:fs/promises";
|
||||
import { extname, join } from "node:path";
|
||||
import {
|
||||
chmod,
|
||||
mkdir,
|
||||
readdir,
|
||||
rm,
|
||||
stat,
|
||||
statfs,
|
||||
utimes,
|
||||
} from "node:fs/promises";
|
||||
import { dirname, extname, join } from "node:path";
|
||||
|
||||
import type { ApiClient } from "../api/client.js";
|
||||
import {
|
||||
@@ -39,12 +47,24 @@ const DIR_MODE = 0o700;
|
||||
const FILE_MODE = 0o600;
|
||||
const TEMP_PREFIX = ".quak-";
|
||||
const TEMP_SUFFIX = ".tmp";
|
||||
const GIB = 1024 * 1024 * 1024;
|
||||
// Owner ruling (#36): bound the originals cache at 100 GiB, but back off when
|
||||
// the volume has under 50 GiB free so the cache never crowds the disk.
|
||||
export const DEFAULT_ORIGINALS_MAX_BYTES = 100 * GIB;
|
||||
export const DEFAULT_FREE_BELOW_BYTES = 50 * GIB;
|
||||
// Ente thumbnails are always JPEG, so the cache stores them with a fixed
|
||||
// extension rather than deriving one from the (image or video) title.
|
||||
const THUMBNAIL_EXT = ".jpg";
|
||||
|
||||
type Kind = "original" | "thumbnail";
|
||||
|
||||
// An original write in progress, with the IDs of the concurrent original writes
|
||||
// it overlaps (recorded both ways as writes begin, cleared when the write ends).
|
||||
interface OriginalWrite {
|
||||
fileID: number;
|
||||
overlaps: Set<number>;
|
||||
}
|
||||
|
||||
// The priority a caller attaches to a thumbnail prefetch. The pool has two
|
||||
// tiers, so this three-value surface collapses onto them: only a currently
|
||||
// visible thumbnail preempts (on-demand); "ahead" prefetch and speculative
|
||||
@@ -133,6 +153,28 @@ export interface CachedPaths {
|
||||
thumbnailPath?: string;
|
||||
}
|
||||
|
||||
// The slice of `fs.statfs` the eviction limit needs: `bavail` is the blocks
|
||||
// available to an unprivileged writer and `bsize` their size, so
|
||||
// `bavail * bsize` is the free byte count. Injectable so tests drive the
|
||||
// adaptive limit without a real volume.
|
||||
export interface StatFsResult {
|
||||
bsize: number;
|
||||
bavail: number;
|
||||
}
|
||||
export type StatFsFn = (path: string) => Promise<StatFsResult>;
|
||||
|
||||
const realStatFs: StatFsFn = async (path) => {
|
||||
const s = await statfs(path);
|
||||
return { bsize: s.bsize, bavail: s.bavail };
|
||||
};
|
||||
|
||||
// The current usage and effective limit of the originals cache, in bytes.
|
||||
// `limitBytes` is the adaptive ceiling last computed (see `originalsLimit`).
|
||||
export interface OriginalsStatus {
|
||||
usedBytes: number;
|
||||
limitBytes?: number;
|
||||
}
|
||||
|
||||
export interface ContentCacheOptions {
|
||||
pools: RequestPools;
|
||||
source: ContentSource;
|
||||
@@ -144,6 +186,18 @@ export interface ContentCacheOptions {
|
||||
// Resolve any membership of a file; every membership shares the underlying
|
||||
// content key, so any one decrypts the same bytes.
|
||||
getFile: (fileID: number) => EnteFile | undefined;
|
||||
// Hard ceiling on `cacheDirectory/originals` (default 100 GiB) and the free
|
||||
// space to protect on the volume (default 50 GiB). The effective limit is
|
||||
// the lesser of the ceiling and what fits above the protected free space.
|
||||
cacheOriginalsMaxBytes?: number;
|
||||
freeBelowBytes?: number;
|
||||
// Whether an original is pinned (favorites + latest week; the precache unit
|
||||
// #48 supplies the set). Pinned originals are never evicted; when only
|
||||
// pinned originals remain the cache runs over-limit until the set shrinks.
|
||||
isPinned?: (fileID: number) => boolean;
|
||||
// Free-space probe on the volume holding `cacheDirectory`; defaults to the
|
||||
// real `fs.statfs`.
|
||||
statfs?: StatFsFn;
|
||||
}
|
||||
|
||||
// Thrown inside a pooled task to drop a queued fetch that was aborted before it
|
||||
@@ -190,6 +244,24 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
// listing at open() and extended as fetches store new files.
|
||||
private readonly originals = new Map<number, string>();
|
||||
private readonly thumbnails = new Map<number, string>();
|
||||
private readonly maxOriginalsBytes: number;
|
||||
private readonly freeBelowBytes: number;
|
||||
private readonly isPinned: (fileID: number) => boolean;
|
||||
private readonly statfs: StatFsFn;
|
||||
// The last measured usage and effective limit, refreshed at open() and after
|
||||
// every original write; exposed through `originalsStatus`.
|
||||
private originalsUsedBytes = 0;
|
||||
private originalsLimitBytes?: number;
|
||||
// Serializes limit enforcement so concurrent original writes never race on
|
||||
// the map or delete each other's just-freed room.
|
||||
private enforcing: Promise<void> = Promise.resolve();
|
||||
// Original writes in progress. Writes for different files run concurrently
|
||||
// (the content pool), so an eviction pass must never delete a file whose
|
||||
// fetch has not yet returned. Each entry records the IDs of the concurrent
|
||||
// original writes it overlaps — noted both ways as writes begin — and a
|
||||
// write's eviction pass spares them all. Bounded by the pool's concurrency,
|
||||
// so eviction is never deferred beyond the active working set.
|
||||
private readonly inFlightOriginals = new Set<OriginalWrite>();
|
||||
|
||||
constructor(opts: ContentCacheOptions) {
|
||||
this.pools = opts.pools;
|
||||
@@ -198,6 +270,11 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
this.getFile = opts.getFile;
|
||||
this.originalsDir = join(opts.cacheDirectory, "originals");
|
||||
this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails");
|
||||
this.maxOriginalsBytes =
|
||||
opts.cacheOriginalsMaxBytes ?? DEFAULT_ORIGINALS_MAX_BYTES;
|
||||
this.freeBelowBytes = opts.freeBelowBytes ?? DEFAULT_FREE_BELOW_BYTES;
|
||||
this.isPinned = opts.isPinned ?? (() => false);
|
||||
this.statfs = opts.statfs ?? realStatFs;
|
||||
}
|
||||
|
||||
// Prepare the cache directories, reap orphan temp files, and take the
|
||||
@@ -207,6 +284,18 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
await this.ensureDir(this.thumbnailsDir);
|
||||
await this.scan(this.originalsDir, this.originals);
|
||||
await this.scan(this.thumbnailsDir, this.thumbnails);
|
||||
// Publish the current usage and limit without evicting; a restart
|
||||
// reuses whatever survived on disk. Eviction only ever fires on a write.
|
||||
await this.refreshOriginalsLimit();
|
||||
}
|
||||
|
||||
// The current originals usage and effective limit, both in bytes, as of the
|
||||
// last write or open. `status().originalsLimitBytes` surfaces this.
|
||||
originalsStatus(): OriginalsStatus {
|
||||
return {
|
||||
usedBytes: this.originalsUsedBytes,
|
||||
limitBytes: this.originalsLimitBytes,
|
||||
};
|
||||
}
|
||||
|
||||
// The cache paths known for a file, for the record projection to expose as
|
||||
@@ -321,8 +410,16 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
const cached = known.get(fileID);
|
||||
if (cached !== undefined) {
|
||||
const size = fileSize(cached);
|
||||
if (size !== undefined && size > 0)
|
||||
if (size !== undefined && size > 0) {
|
||||
// Returning an original's path is a use: bump its mtime so LRU
|
||||
// order reflects it and survives a restart with no ledger.
|
||||
if (
|
||||
kind === "original" &&
|
||||
dirname(cached) === this.originalsDir
|
||||
)
|
||||
await this.touch(cached);
|
||||
return { path: cached, bytes: size, cached: true };
|
||||
}
|
||||
// A recorded file that has since gone re-fetches below.
|
||||
known.delete(fileID);
|
||||
}
|
||||
@@ -358,6 +455,14 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
// to completion.
|
||||
if (signal?.aborted) throw new AbortDrop();
|
||||
|
||||
// Register this original among those in flight, linking it with
|
||||
// every sibling already writing so neither evicts the other's
|
||||
// file. Non-null iff this is an original.
|
||||
const write =
|
||||
kind === "original"
|
||||
? this.beginOriginalWrite(fileID)
|
||||
: null;
|
||||
try {
|
||||
await this.download(file, dest, kind, opts?.onByte);
|
||||
await chmod(dest, FILE_MODE);
|
||||
const size = (await stat(dest)).size;
|
||||
@@ -367,7 +472,15 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
);
|
||||
}
|
||||
known.set(fileID, dest);
|
||||
// A fresh original may have crossed the limit; make room by
|
||||
// evicting least-recently-used originals. An over-budget
|
||||
// fetch keeps the file it returns, and no overlapping
|
||||
// sibling is evicted. Thumbnails are never bounded.
|
||||
if (write) await this.enforceOriginalsLimit(write);
|
||||
return { path: dest, bytes: size, cached: false };
|
||||
} finally {
|
||||
if (write) this.inFlightOriginals.delete(write);
|
||||
}
|
||||
},
|
||||
{ priority, key: fileID },
|
||||
);
|
||||
@@ -387,6 +500,124 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
return result.bytesWritten;
|
||||
}
|
||||
|
||||
// Best-effort bump of a file's mtime to now; a failed touch must never fail
|
||||
// the read it accompanies.
|
||||
private async touch(path: string): Promise<void> {
|
||||
const now = new Date();
|
||||
await utimes(path, now, now).catch(() => undefined);
|
||||
}
|
||||
|
||||
// Every stored original that lives under `originalsDir` (a backup-directory
|
||||
// hit recorded in the map is excluded), with its size and mtime. Entries
|
||||
// whose file has vanished are dropped from the map. Backups and thumbnails
|
||||
// are never counted.
|
||||
private async measureOriginals(): Promise<{
|
||||
entries: {
|
||||
fileID: number;
|
||||
path: string;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
}[];
|
||||
used: number;
|
||||
}> {
|
||||
const entries: {
|
||||
fileID: number;
|
||||
path: string;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
}[] = [];
|
||||
let used = 0;
|
||||
for (const [fileID, path] of this.originals) {
|
||||
if (dirname(path) !== this.originalsDir) continue;
|
||||
try {
|
||||
const s = await stat(path);
|
||||
entries.push({
|
||||
fileID,
|
||||
path,
|
||||
size: s.size,
|
||||
mtimeMs: s.mtimeMs,
|
||||
});
|
||||
used += s.size;
|
||||
} catch {
|
||||
this.originals.delete(fileID);
|
||||
}
|
||||
}
|
||||
return { entries, used };
|
||||
}
|
||||
|
||||
// The effective ceiling on originals: the configured max, but no more than
|
||||
// what fits once the protected free space is set aside. `used + free` is the
|
||||
// volume space the cache could occupy; subtracting `freeBelowBytes` leaves
|
||||
// the reserve untouched. Clamped at zero.
|
||||
private async originalsLimit(used: number): Promise<number> {
|
||||
const { bsize, bavail } = await this.statfs(this.originalsDir);
|
||||
const free = bsize * bavail;
|
||||
const adaptive = used + free - this.freeBelowBytes;
|
||||
return Math.max(0, Math.min(this.maxOriginalsBytes, adaptive));
|
||||
}
|
||||
|
||||
// Recompute and publish usage and limit without evicting (used at open()).
|
||||
private refreshOriginalsLimit(): Promise<void> {
|
||||
return this.serializeEnforce(async () => {
|
||||
const { used } = await this.measureOriginals();
|
||||
this.originalsUsedBytes = used;
|
||||
this.originalsLimitBytes = await this.originalsLimit(used);
|
||||
});
|
||||
}
|
||||
|
||||
// Record a starting original write among those in flight, linking it with
|
||||
// every sibling already writing so neither can evict the other's file.
|
||||
private beginOriginalWrite(fileID: number): OriginalWrite {
|
||||
const write: OriginalWrite = { fileID, overlaps: new Set() };
|
||||
for (const other of this.inFlightOriginals) {
|
||||
write.overlaps.add(other.fileID);
|
||||
other.overlaps.add(fileID);
|
||||
}
|
||||
this.inFlightOriginals.add(write);
|
||||
return write;
|
||||
}
|
||||
|
||||
// Evict least-recently-used originals until usage fits the limit. Skipped:
|
||||
// pinned originals, the file `write` just stored, and every original whose
|
||||
// write overlaps it (`write.overlaps`). The last two spare any fetch whose
|
||||
// lifetime overlaps this one, so concurrent over-budget fetches all keep the
|
||||
// paths they return; when only such originals remain the cache stays
|
||||
// over-limit until they settle, and a later, non-overlapping write finds
|
||||
// them eligible again.
|
||||
private enforceOriginalsLimit(write: OriginalWrite): Promise<void> {
|
||||
return this.serializeEnforce(async () => {
|
||||
const { entries, used } = await this.measureOriginals();
|
||||
const limit = await this.originalsLimit(used);
|
||||
let remaining = used;
|
||||
if (remaining > limit) {
|
||||
const evictable = entries
|
||||
.filter(
|
||||
(e) =>
|
||||
e.fileID !== write.fileID &&
|
||||
!write.overlaps.has(e.fileID) &&
|
||||
!this.isPinned(e.fileID),
|
||||
)
|
||||
.sort((a, b) => a.mtimeMs - b.mtimeMs);
|
||||
for (const e of evictable) {
|
||||
if (remaining <= limit) break;
|
||||
await rm(e.path, { force: true });
|
||||
this.originals.delete(e.fileID);
|
||||
remaining -= e.size;
|
||||
}
|
||||
}
|
||||
this.originalsUsedBytes = remaining;
|
||||
this.originalsLimitBytes = limit;
|
||||
});
|
||||
}
|
||||
|
||||
// Run limit work one at a time; failures are swallowed so a transient
|
||||
// statfs or unlink error never rejects the read or write that triggered it.
|
||||
private serializeEnforce(work: () => Promise<void>): Promise<void> {
|
||||
const next = this.enforcing.then(work).catch(() => undefined);
|
||||
this.enforcing = next;
|
||||
return next;
|
||||
}
|
||||
|
||||
private async ensureDir(dir: string): Promise<void> {
|
||||
// chmod after mkdir so the mode is tightened even when the directory
|
||||
// already existed with a looser one; mkdir alone would not.
|
||||
|
||||
@@ -132,6 +132,14 @@ export interface LibraryOptions {
|
||||
// Overrides the client's own `contentSource()`; mainly for tests that drive
|
||||
// the cache with a stand-in source.
|
||||
contentSource?: ContentSource;
|
||||
// Bound on `cacheDirectory/originals` (default 100 GiB) and the free space
|
||||
// to protect on its volume (default 50 GiB). The effective limit adapts
|
||||
// down as the disk fills; `status().originalsLimitBytes` reports it.
|
||||
cacheOriginalsMaxBytes?: number;
|
||||
freeBelowBytes?: number;
|
||||
// Whether an original is pinned and so never evicted (favorites + latest
|
||||
// week; the precache unit #48 supplies the set).
|
||||
isOriginalPinned?: (fileID: number) => boolean;
|
||||
}
|
||||
|
||||
export interface LibraryStatus {
|
||||
@@ -153,6 +161,10 @@ export interface LibraryStatus {
|
||||
// when ML fetching is disabled.
|
||||
mlStored?: number;
|
||||
mlIndexed?: number;
|
||||
// Bytes stored in the originals cache and the effective size limit as of the
|
||||
// last write or open; undefined when no content cache is open.
|
||||
originalsUsedBytes?: number;
|
||||
originalsLimitBytes?: number;
|
||||
closed: boolean;
|
||||
}
|
||||
|
||||
@@ -289,6 +301,9 @@ export class Library {
|
||||
cacheDirectory,
|
||||
downloadDirectory: opts.downloadDirectory,
|
||||
getFile: (fileID) => store.getFileByID(fileID),
|
||||
cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes,
|
||||
freeBelowBytes: opts.freeBelowBytes,
|
||||
isPinned: opts.isOriginalPinned,
|
||||
});
|
||||
await cache.open();
|
||||
}
|
||||
@@ -364,6 +379,7 @@ export class Library {
|
||||
files += this.store.listFiles(c.id).length;
|
||||
}
|
||||
const ml = this.mldata?.stats();
|
||||
const originals = this.cache?.originalsStatus();
|
||||
return {
|
||||
userID: this.store.userID,
|
||||
collections: collections.length,
|
||||
@@ -374,6 +390,8 @@ export class Library {
|
||||
lastMLError: this.lastMLError,
|
||||
mlStored: ml?.stored,
|
||||
mlIndexed: ml?.indexed,
|
||||
originalsUsedBytes: originals?.usedBytes,
|
||||
originalsLimitBytes: originals?.limitBytes,
|
||||
closed: this.closed,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
/**
|
||||
* Tests for the originals cache size limit and LRU eviction (issue #47),
|
||||
* layered on the on-disk content cache (#46).
|
||||
*
|
||||
* Only `cacheDirectory/originals` is bounded and evicted. Before each original
|
||||
* write the effective limit is
|
||||
* min(cacheOriginalsMaxBytes, bytesUsedByOriginals + bytesFree - freeBelowBytes)
|
||||
* with `bytesFree` read from `statfs` on the volume holding `cacheDirectory`.
|
||||
* The limit therefore falls as the disk fills and rises as space returns. When
|
||||
* a write would cross the limit, least-recently-used originals are removed until
|
||||
* it fits; pinned files are skipped, and if only pinned files remain the write
|
||||
* proceeds over-limit. Last-use is the file `mtime`, bumped whenever a read
|
||||
* returns an original's path, so ordering survives a restart with no ledger.
|
||||
*
|
||||
* `statfs` is injected so the adaptive limit is exercised deterministically:
|
||||
* `bsize` is 1, so `bavail` is the free byte count the formula sees. The source
|
||||
* writes a controllable number of bytes per file, and tests set each stored
|
||||
* file's `mtime` explicitly so LRU order does not depend on wall-clock timing.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdtempSync, rmSync, existsSync, statSync, utimesSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import {
|
||||
ContentCache,
|
||||
type ContentSource,
|
||||
type StatFsFn,
|
||||
} from "../../src/library/content.js";
|
||||
import { RequestPools } from "../../src/library/pools.js";
|
||||
import type { EnteFile } from "../../src/model/types.js";
|
||||
|
||||
const file = (id: number): EnteFile => ({
|
||||
id,
|
||||
collectionID: 1,
|
||||
ownerID: 1,
|
||||
key: new Uint8Array([id & 0xff]),
|
||||
metadata: {
|
||||
title: `file-${id}.jpg`,
|
||||
fileType: "image",
|
||||
creationTime: 0,
|
||||
modificationTime: 0,
|
||||
},
|
||||
file: { decryptionHeader: "aGVhZGVy" },
|
||||
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
||||
updationTime: 0,
|
||||
});
|
||||
|
||||
// A source that writes a controllable number of bytes per file (default 10).
|
||||
class SizedSource implements ContentSource {
|
||||
sizeFor = new Map<number, number>();
|
||||
|
||||
async original(args: {
|
||||
file: EnteFile;
|
||||
destination: string;
|
||||
}): Promise<{ bytesWritten: number }> {
|
||||
const n = this.sizeFor.get(args.file.id) ?? 10;
|
||||
await writeFile(args.destination, Buffer.alloc(n, 1));
|
||||
return { bytesWritten: n };
|
||||
}
|
||||
|
||||
async thumbnail(args: {
|
||||
file: EnteFile;
|
||||
destination: string;
|
||||
}): Promise<{ bytesWritten: number }> {
|
||||
await writeFile(args.destination, Buffer.alloc(10, 1));
|
||||
return { bytesWritten: 10 };
|
||||
}
|
||||
}
|
||||
|
||||
// A source whose original() writes 10 bytes but blocks on a gate before
|
||||
// returning, so two fetches can be held in flight together. `bothStarted`
|
||||
// resolves once both originals have entered, and `release()` lets them finish.
|
||||
class GatedSource implements ContentSource {
|
||||
private openGate!: () => void;
|
||||
private readonly gate = new Promise<void>((r) => (this.openGate = r));
|
||||
private inFlight = 0;
|
||||
private reachedTwo!: () => void;
|
||||
readonly bothStarted = new Promise<void>((r) => (this.reachedTwo = r));
|
||||
|
||||
async original(args: {
|
||||
file: EnteFile;
|
||||
destination: string;
|
||||
}): Promise<{ bytesWritten: number }> {
|
||||
this.inFlight += 1;
|
||||
if (this.inFlight === 2) this.reachedTwo();
|
||||
await this.gate;
|
||||
await writeFile(args.destination, Buffer.alloc(10, 1));
|
||||
return { bytesWritten: 10 };
|
||||
}
|
||||
|
||||
async thumbnail(args: {
|
||||
file: EnteFile;
|
||||
destination: string;
|
||||
}): Promise<{ bytesWritten: number }> {
|
||||
await writeFile(args.destination, Buffer.alloc(10, 1));
|
||||
return { bytesWritten: 10 };
|
||||
}
|
||||
|
||||
release(): void {
|
||||
this.openGate();
|
||||
}
|
||||
}
|
||||
|
||||
let root: string;
|
||||
let cacheDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
root = mkdtempSync(join(tmpdir(), "quak-eviction-"));
|
||||
cacheDir = join(root, "cache");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (root && existsSync(root))
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const originalPath = (id: number): string =>
|
||||
join(cacheDir, "originals", `${id}.jpg`);
|
||||
|
||||
// Pin an original's mtime to a fixed second-resolution instant so LRU order is
|
||||
// deterministic. Lower `seconds` = older = evicted first.
|
||||
const setMtime = (id: number, seconds: number): void => {
|
||||
utimesSync(originalPath(id), seconds, seconds);
|
||||
};
|
||||
|
||||
const buildCache = (args: {
|
||||
source?: ContentSource;
|
||||
files?: EnteFile[];
|
||||
statfs: StatFsFn;
|
||||
cacheOriginalsMaxBytes?: number;
|
||||
freeBelowBytes?: number;
|
||||
isPinned?: (fileID: number) => boolean;
|
||||
}): { cache: ContentCache; source: SizedSource } => {
|
||||
const source = (args.source as SizedSource) ?? new SizedSource();
|
||||
const byID = new Map<number, EnteFile>();
|
||||
for (const f of args.files ?? [file(1), file(2), file(3), file(4), file(5)])
|
||||
byID.set(f.id, f);
|
||||
const cache = new ContentCache({
|
||||
pools: new RequestPools(),
|
||||
source,
|
||||
cacheDirectory: cacheDir,
|
||||
getFile: (id) => byID.get(id),
|
||||
statfs: args.statfs,
|
||||
cacheOriginalsMaxBytes: args.cacheOriginalsMaxBytes,
|
||||
freeBelowBytes: args.freeBelowBytes,
|
||||
isPinned: args.isPinned,
|
||||
});
|
||||
return { cache, source };
|
||||
};
|
||||
|
||||
// Plenty of free space, so the configured max governs the limit unless a test
|
||||
// dials it down. `bsize` of 1 makes `bavail` the free byte count.
|
||||
const abundantFree: StatFsFn = async () => ({
|
||||
bsize: 1,
|
||||
bavail: 1_000_000_000,
|
||||
});
|
||||
|
||||
describe("originals eviction", () => {
|
||||
it("removes least-recently-used originals to fit the limit", async () => {
|
||||
// Each original is 10 bytes; a 25-byte cap holds two.
|
||||
const { cache } = buildCache({
|
||||
statfs: abundantFree,
|
||||
cacheOriginalsMaxBytes: 25,
|
||||
freeBelowBytes: 0,
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
await cache.original(1);
|
||||
setMtime(1, 1000);
|
||||
await cache.original(2);
|
||||
setMtime(2, 2000);
|
||||
// Writing the third (total 30 > 25) evicts the oldest, file 1.
|
||||
await cache.original(3);
|
||||
|
||||
expect(existsSync(originalPath(1))).toBe(false);
|
||||
expect(existsSync(originalPath(2))).toBe(true);
|
||||
expect(existsSync(originalPath(3))).toBe(true);
|
||||
expect(cache.pathsFor(1).originalPath).toBeUndefined();
|
||||
expect(cache.originalsStatus().usedBytes).toBe(20);
|
||||
});
|
||||
|
||||
it("skips pinned files, evicting the oldest unpinned instead", async () => {
|
||||
const { cache } = buildCache({
|
||||
statfs: abundantFree,
|
||||
cacheOriginalsMaxBytes: 25,
|
||||
freeBelowBytes: 0,
|
||||
isPinned: (id) => id === 1,
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
await cache.original(1);
|
||||
setMtime(1, 1000); // oldest, but pinned
|
||||
await cache.original(2);
|
||||
setMtime(2, 2000);
|
||||
await cache.original(3);
|
||||
|
||||
// File 1 is oldest but pinned, so file 2 is evicted instead.
|
||||
expect(existsSync(originalPath(1))).toBe(true);
|
||||
expect(existsSync(originalPath(2))).toBe(false);
|
||||
expect(existsSync(originalPath(3))).toBe(true);
|
||||
});
|
||||
|
||||
it("proceeds over-limit when only pinned files remain", async () => {
|
||||
// A 5-byte cap cannot hold even one 10-byte original.
|
||||
const { cache } = buildCache({
|
||||
statfs: abundantFree,
|
||||
cacheOriginalsMaxBytes: 5,
|
||||
freeBelowBytes: 0,
|
||||
isPinned: () => true,
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
const result = await cache.original(1);
|
||||
|
||||
expect(existsSync(result.path)).toBe(true);
|
||||
const status = cache.originalsStatus();
|
||||
expect(status.usedBytes).toBe(10);
|
||||
expect(status.usedBytes).toBeGreaterThan(status.limitBytes ?? 0);
|
||||
});
|
||||
|
||||
it("keeps a just-written original larger than the limit, evicting it only on a later write", async () => {
|
||||
// A 5-byte cap cannot hold even one 10-byte original, but a fetch must
|
||||
// never evict the file it just wrote and is about to return.
|
||||
const { cache } = buildCache({
|
||||
statfs: abundantFree,
|
||||
cacheOriginalsMaxBytes: 5,
|
||||
freeBelowBytes: 0,
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
const result = await cache.original(1);
|
||||
|
||||
// The just-written original survives its own over-budget write and the
|
||||
// returned path exists on disk.
|
||||
expect(existsSync(result.path)).toBe(true);
|
||||
expect(existsSync(originalPath(1))).toBe(true);
|
||||
expect(cache.originalsStatus().usedBytes).toBe(10);
|
||||
setMtime(1, 1000);
|
||||
|
||||
// A later write finds file 1 eligible and evicts it to make room, while
|
||||
// the newly written file 2 is itself kept over-limit.
|
||||
await cache.original(2);
|
||||
expect(existsSync(originalPath(1))).toBe(false);
|
||||
expect(existsSync(originalPath(2))).toBe(true);
|
||||
expect(cache.originalsStatus().usedBytes).toBe(10);
|
||||
});
|
||||
|
||||
it("adapts the limit down as the disk fills and up as space returns", async () => {
|
||||
// The configured max is generous; free space drives the limit. Eviction
|
||||
// only kicks in once free space falls below the protected reserve.
|
||||
let free = 1_000_000;
|
||||
const statfs: StatFsFn = async () => ({ bsize: 1, bavail: free });
|
||||
const { cache } = buildCache({
|
||||
statfs,
|
||||
cacheOriginalsMaxBytes: 1000,
|
||||
freeBelowBytes: 100,
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
// Ample free space: the limit is the configured max and nothing is
|
||||
// evicted as three 10-byte originals accumulate.
|
||||
await cache.original(1);
|
||||
setMtime(1, 1000);
|
||||
await cache.original(2);
|
||||
setMtime(2, 2000);
|
||||
await cache.original(3);
|
||||
setMtime(3, 3000);
|
||||
expect(cache.originalsStatus().limitBytes).toBe(1000);
|
||||
expect(cache.originalsStatus().usedBytes).toBe(30);
|
||||
|
||||
// The disk fills: only 95 bytes free, below the 100-byte reserve. The
|
||||
// next write (used 40) sees limit = min(1000, 40 + 95 - 100) = 35 and
|
||||
// evicts the oldest original (file 1) to fit.
|
||||
free = 95;
|
||||
await cache.original(4);
|
||||
expect(cache.originalsStatus().limitBytes).toBe(35);
|
||||
expect(existsSync(originalPath(1))).toBe(false);
|
||||
expect(cache.originalsStatus().usedBytes).toBe(30);
|
||||
setMtime(4, 4000);
|
||||
|
||||
// Space returns: the limit rises back to the configured max and the
|
||||
// next write is kept without eviction.
|
||||
free = 1_000_000;
|
||||
await cache.original(5);
|
||||
expect(cache.originalsStatus().limitBytes).toBe(1000);
|
||||
expect(existsSync(originalPath(4))).toBe(true);
|
||||
expect(existsSync(originalPath(5))).toBe(true);
|
||||
});
|
||||
|
||||
it("bumps an original's mtime when a read returns its path", async () => {
|
||||
const { cache } = buildCache({
|
||||
statfs: abundantFree,
|
||||
cacheOriginalsMaxBytes: 1000,
|
||||
freeBelowBytes: 0,
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
await cache.original(1);
|
||||
// Age the file well into the past.
|
||||
setMtime(1, 1000);
|
||||
expect(statSync(originalPath(1)).mtimeMs).toBeLessThan(2_000_000);
|
||||
|
||||
// A second read is a cache hit that must touch the file.
|
||||
const before = Date.now();
|
||||
await cache.original(1);
|
||||
expect(statSync(originalPath(1)).mtimeMs).toBeGreaterThanOrEqual(
|
||||
before - 2000,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps both originals when two over-budget fetches race", async () => {
|
||||
// A 15-byte cap holds one 10-byte original but not two. Two fetches are
|
||||
// held in flight together; when both stored files cross the limit,
|
||||
// neither eviction pass may delete the sibling whose path has not yet
|
||||
// been returned, so both survive over-limit.
|
||||
const source = new GatedSource();
|
||||
const { cache } = buildCache({
|
||||
source,
|
||||
statfs: abundantFree,
|
||||
cacheOriginalsMaxBytes: 15,
|
||||
freeBelowBytes: 0,
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
const p1 = cache.original(1);
|
||||
const p2 = cache.original(2);
|
||||
await source.bothStarted; // both downloads are in flight before either stores
|
||||
source.release();
|
||||
const [r1, r2] = await Promise.all([p1, p2]);
|
||||
|
||||
// Both returned paths exist on disk even though together they exceed the
|
||||
// cap; nothing was evicted out from under a fetch still in progress.
|
||||
expect(existsSync(r1.path)).toBe(true);
|
||||
expect(existsSync(r2.path)).toBe(true);
|
||||
expect(existsSync(originalPath(1))).toBe(true);
|
||||
expect(existsSync(originalPath(2))).toBe(true);
|
||||
expect(cache.originalsStatus().usedBytes).toBe(20);
|
||||
});
|
||||
|
||||
it("never counts or evicts thumbnails", async () => {
|
||||
const { cache } = buildCache({
|
||||
statfs: abundantFree,
|
||||
cacheOriginalsMaxBytes: 5,
|
||||
freeBelowBytes: 0,
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
await cache.thumbnail(1);
|
||||
await cache.thumbnail(2);
|
||||
|
||||
expect(cache.pathsFor(1).thumbnailPath).toBeDefined();
|
||||
expect(cache.pathsFor(2).thumbnailPath).toBeDefined();
|
||||
expect(cache.originalsStatus().usedBytes).toBe(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user