Compare commits
1
Commits
e7f77b4b27
...
9d09a7db01
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d09a7db01 |
+197
-3
@@ -23,8 +23,16 @@
|
|||||||
// decrypted length this layer has.
|
// decrypted length this layer has.
|
||||||
|
|
||||||
import { existsSync, statSync } from "node:fs";
|
import { existsSync, statSync } from "node:fs";
|
||||||
import { chmod, mkdir, readdir, rm, stat } from "node:fs/promises";
|
import {
|
||||||
import { extname, join } from "node:path";
|
chmod,
|
||||||
|
mkdir,
|
||||||
|
readdir,
|
||||||
|
rm,
|
||||||
|
stat,
|
||||||
|
statfs,
|
||||||
|
utimes,
|
||||||
|
} from "node:fs/promises";
|
||||||
|
import { dirname, extname, join } from "node:path";
|
||||||
|
|
||||||
import type { ApiClient } from "../api/client.js";
|
import type { ApiClient } from "../api/client.js";
|
||||||
import {
|
import {
|
||||||
@@ -39,6 +47,11 @@ const DIR_MODE = 0o700;
|
|||||||
const FILE_MODE = 0o600;
|
const FILE_MODE = 0o600;
|
||||||
const TEMP_PREFIX = ".quak-";
|
const TEMP_PREFIX = ".quak-";
|
||||||
const TEMP_SUFFIX = ".tmp";
|
const TEMP_SUFFIX = ".tmp";
|
||||||
|
const GIB = 1024 * 1024 * 1024;
|
||||||
|
// Owner ruling (#36): bound the originals cache at 100 GiB, but back off when
|
||||||
|
// the volume has under 50 GiB free so the cache never crowds the disk.
|
||||||
|
export const DEFAULT_ORIGINALS_MAX_BYTES = 100 * GIB;
|
||||||
|
export const DEFAULT_FREE_BELOW_BYTES = 50 * GIB;
|
||||||
// Ente thumbnails are always JPEG, so the cache stores them with a fixed
|
// Ente thumbnails are always JPEG, so the cache stores them with a fixed
|
||||||
// extension rather than deriving one from the (image or video) title.
|
// extension rather than deriving one from the (image or video) title.
|
||||||
const THUMBNAIL_EXT = ".jpg";
|
const THUMBNAIL_EXT = ".jpg";
|
||||||
@@ -133,6 +146,28 @@ export interface CachedPaths {
|
|||||||
thumbnailPath?: string;
|
thumbnailPath?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The slice of `fs.statfs` the eviction limit needs: `bavail` is the blocks
|
||||||
|
// available to an unprivileged writer and `bsize` their size, so
|
||||||
|
// `bavail * bsize` is the free byte count. Injectable so tests drive the
|
||||||
|
// adaptive limit without a real volume.
|
||||||
|
export interface StatFsResult {
|
||||||
|
bsize: number;
|
||||||
|
bavail: number;
|
||||||
|
}
|
||||||
|
export type StatFsFn = (path: string) => Promise<StatFsResult>;
|
||||||
|
|
||||||
|
const realStatFs: StatFsFn = async (path) => {
|
||||||
|
const s = await statfs(path);
|
||||||
|
return { bsize: s.bsize, bavail: s.bavail };
|
||||||
|
};
|
||||||
|
|
||||||
|
// The current usage and effective limit of the originals cache, in bytes.
|
||||||
|
// `limitBytes` is the adaptive ceiling last computed (see `originalsLimit`).
|
||||||
|
export interface OriginalsStatus {
|
||||||
|
usedBytes: number;
|
||||||
|
limitBytes?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface ContentCacheOptions {
|
export interface ContentCacheOptions {
|
||||||
pools: RequestPools;
|
pools: RequestPools;
|
||||||
source: ContentSource;
|
source: ContentSource;
|
||||||
@@ -144,6 +179,18 @@ export interface ContentCacheOptions {
|
|||||||
// Resolve any membership of a file; every membership shares the underlying
|
// Resolve any membership of a file; every membership shares the underlying
|
||||||
// content key, so any one decrypts the same bytes.
|
// content key, so any one decrypts the same bytes.
|
||||||
getFile: (fileID: number) => EnteFile | undefined;
|
getFile: (fileID: number) => EnteFile | undefined;
|
||||||
|
// Hard ceiling on `cacheDirectory/originals` (default 100 GiB) and the free
|
||||||
|
// space to protect on the volume (default 50 GiB). The effective limit is
|
||||||
|
// the lesser of the ceiling and what fits above the protected free space.
|
||||||
|
cacheOriginalsMaxBytes?: number;
|
||||||
|
freeBelowBytes?: number;
|
||||||
|
// Whether an original is pinned (favorites + latest week; the precache unit
|
||||||
|
// #48 supplies the set). Pinned originals are never evicted; when only
|
||||||
|
// pinned originals remain the cache runs over-limit until the set shrinks.
|
||||||
|
isPinned?: (fileID: number) => boolean;
|
||||||
|
// Free-space probe on the volume holding `cacheDirectory`; defaults to the
|
||||||
|
// real `fs.statfs`.
|
||||||
|
statfs?: StatFsFn;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Thrown inside a pooled task to drop a queued fetch that was aborted before it
|
// Thrown inside a pooled task to drop a queued fetch that was aborted before it
|
||||||
@@ -190,6 +237,17 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
// listing at open() and extended as fetches store new files.
|
// listing at open() and extended as fetches store new files.
|
||||||
private readonly originals = new Map<number, string>();
|
private readonly originals = new Map<number, string>();
|
||||||
private readonly thumbnails = new Map<number, string>();
|
private readonly thumbnails = new Map<number, string>();
|
||||||
|
private readonly maxOriginalsBytes: number;
|
||||||
|
private readonly freeBelowBytes: number;
|
||||||
|
private readonly isPinned: (fileID: number) => boolean;
|
||||||
|
private readonly statfs: StatFsFn;
|
||||||
|
// The last measured usage and effective limit, refreshed at open() and after
|
||||||
|
// every original write; exposed through `originalsStatus`.
|
||||||
|
private originalsUsedBytes = 0;
|
||||||
|
private originalsLimitBytes?: number;
|
||||||
|
// Serializes limit enforcement so concurrent original writes never race on
|
||||||
|
// the map or delete each other's just-freed room.
|
||||||
|
private enforcing: Promise<void> = Promise.resolve();
|
||||||
|
|
||||||
constructor(opts: ContentCacheOptions) {
|
constructor(opts: ContentCacheOptions) {
|
||||||
this.pools = opts.pools;
|
this.pools = opts.pools;
|
||||||
@@ -198,6 +256,11 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
this.getFile = opts.getFile;
|
this.getFile = opts.getFile;
|
||||||
this.originalsDir = join(opts.cacheDirectory, "originals");
|
this.originalsDir = join(opts.cacheDirectory, "originals");
|
||||||
this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails");
|
this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails");
|
||||||
|
this.maxOriginalsBytes =
|
||||||
|
opts.cacheOriginalsMaxBytes ?? DEFAULT_ORIGINALS_MAX_BYTES;
|
||||||
|
this.freeBelowBytes = opts.freeBelowBytes ?? DEFAULT_FREE_BELOW_BYTES;
|
||||||
|
this.isPinned = opts.isPinned ?? (() => false);
|
||||||
|
this.statfs = opts.statfs ?? realStatFs;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare the cache directories, reap orphan temp files, and take the
|
// Prepare the cache directories, reap orphan temp files, and take the
|
||||||
@@ -207,6 +270,18 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
await this.ensureDir(this.thumbnailsDir);
|
await this.ensureDir(this.thumbnailsDir);
|
||||||
await this.scan(this.originalsDir, this.originals);
|
await this.scan(this.originalsDir, this.originals);
|
||||||
await this.scan(this.thumbnailsDir, this.thumbnails);
|
await this.scan(this.thumbnailsDir, this.thumbnails);
|
||||||
|
// Publish the current usage and limit without evicting; a restart
|
||||||
|
// reuses whatever survived on disk. Eviction only ever fires on a write.
|
||||||
|
await this.refreshOriginalsLimit();
|
||||||
|
}
|
||||||
|
|
||||||
|
// The current originals usage and effective limit, both in bytes, as of the
|
||||||
|
// last write or open. `status().originalsLimitBytes` surfaces this.
|
||||||
|
originalsStatus(): OriginalsStatus {
|
||||||
|
return {
|
||||||
|
usedBytes: this.originalsUsedBytes,
|
||||||
|
limitBytes: this.originalsLimitBytes,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// The cache paths known for a file, for the record projection to expose as
|
// The cache paths known for a file, for the record projection to expose as
|
||||||
@@ -321,8 +396,16 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
const cached = known.get(fileID);
|
const cached = known.get(fileID);
|
||||||
if (cached !== undefined) {
|
if (cached !== undefined) {
|
||||||
const size = fileSize(cached);
|
const size = fileSize(cached);
|
||||||
if (size !== undefined && size > 0)
|
if (size !== undefined && size > 0) {
|
||||||
|
// Returning an original's path is a use: bump its mtime so LRU
|
||||||
|
// order reflects it and survives a restart with no ledger.
|
||||||
|
if (
|
||||||
|
kind === "original" &&
|
||||||
|
dirname(cached) === this.originalsDir
|
||||||
|
)
|
||||||
|
await this.touch(cached);
|
||||||
return { path: cached, bytes: size, cached: true };
|
return { path: cached, bytes: size, cached: true };
|
||||||
|
}
|
||||||
// A recorded file that has since gone re-fetches below.
|
// A recorded file that has since gone re-fetches below.
|
||||||
known.delete(fileID);
|
known.delete(fileID);
|
||||||
}
|
}
|
||||||
@@ -367,6 +450,13 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
known.set(fileID, dest);
|
known.set(fileID, dest);
|
||||||
|
// A fresh original may have crossed the limit; make room by
|
||||||
|
// evicting least-recently-used originals. The file just written
|
||||||
|
// is excluded from the candidates so an on-demand fetch that is
|
||||||
|
// itself over budget proceeds over-limit and returns a path that
|
||||||
|
// exists on disk. Thumbnails are never bounded or evicted.
|
||||||
|
if (kind === "original")
|
||||||
|
await this.enforceOriginalsLimit(fileID);
|
||||||
return { path: dest, bytes: size, cached: false };
|
return { path: dest, bytes: size, cached: false };
|
||||||
},
|
},
|
||||||
{ priority, key: fileID },
|
{ priority, key: fileID },
|
||||||
@@ -387,6 +477,110 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
return result.bytesWritten;
|
return result.bytesWritten;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Best-effort bump of a file's mtime to now; a failed touch must never fail
|
||||||
|
// the read it accompanies.
|
||||||
|
private async touch(path: string): Promise<void> {
|
||||||
|
const now = new Date();
|
||||||
|
await utimes(path, now, now).catch(() => undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every stored original that lives under `originalsDir` (a backup-directory
|
||||||
|
// hit recorded in the map is excluded), with its size and mtime. Entries
|
||||||
|
// whose file has vanished are dropped from the map. Backups and thumbnails
|
||||||
|
// are never counted.
|
||||||
|
private async measureOriginals(): Promise<{
|
||||||
|
entries: {
|
||||||
|
fileID: number;
|
||||||
|
path: string;
|
||||||
|
size: number;
|
||||||
|
mtimeMs: number;
|
||||||
|
}[];
|
||||||
|
used: number;
|
||||||
|
}> {
|
||||||
|
const entries: {
|
||||||
|
fileID: number;
|
||||||
|
path: string;
|
||||||
|
size: number;
|
||||||
|
mtimeMs: number;
|
||||||
|
}[] = [];
|
||||||
|
let used = 0;
|
||||||
|
for (const [fileID, path] of this.originals) {
|
||||||
|
if (dirname(path) !== this.originalsDir) continue;
|
||||||
|
try {
|
||||||
|
const s = await stat(path);
|
||||||
|
entries.push({
|
||||||
|
fileID,
|
||||||
|
path,
|
||||||
|
size: s.size,
|
||||||
|
mtimeMs: s.mtimeMs,
|
||||||
|
});
|
||||||
|
used += s.size;
|
||||||
|
} catch {
|
||||||
|
this.originals.delete(fileID);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { entries, used };
|
||||||
|
}
|
||||||
|
|
||||||
|
// The effective ceiling on originals: the configured max, but no more than
|
||||||
|
// what fits once the protected free space is set aside. `used + free` is the
|
||||||
|
// volume space the cache could occupy; subtracting `freeBelowBytes` leaves
|
||||||
|
// the reserve untouched. Clamped at zero.
|
||||||
|
private async originalsLimit(used: number): Promise<number> {
|
||||||
|
const { bsize, bavail } = await this.statfs(this.originalsDir);
|
||||||
|
const free = bsize * bavail;
|
||||||
|
const adaptive = used + free - this.freeBelowBytes;
|
||||||
|
return Math.max(0, Math.min(this.maxOriginalsBytes, adaptive));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recompute and publish usage and limit without evicting (used at open()).
|
||||||
|
private refreshOriginalsLimit(): Promise<void> {
|
||||||
|
return this.serializeEnforce(async () => {
|
||||||
|
const { used } = await this.measureOriginals();
|
||||||
|
this.originalsUsedBytes = used;
|
||||||
|
this.originalsLimitBytes = await this.originalsLimit(used);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Evict least-recently-used originals until usage fits the limit. Pinned
|
||||||
|
// originals are skipped, as is `justWrittenID` (the original this write just
|
||||||
|
// stored, so an over-budget on-demand fetch never evicts the file it is
|
||||||
|
// about to return). When only such originals remain the cache stays
|
||||||
|
// over-limit until the protected set shrinks; the skipped file becomes
|
||||||
|
// eligible on a later write.
|
||||||
|
private enforceOriginalsLimit(justWrittenID: number): Promise<void> {
|
||||||
|
return this.serializeEnforce(async () => {
|
||||||
|
const { entries, used } = await this.measureOriginals();
|
||||||
|
const limit = await this.originalsLimit(used);
|
||||||
|
let remaining = used;
|
||||||
|
if (remaining > limit) {
|
||||||
|
const evictable = entries
|
||||||
|
.filter(
|
||||||
|
(e) =>
|
||||||
|
e.fileID !== justWrittenID &&
|
||||||
|
!this.isPinned(e.fileID),
|
||||||
|
)
|
||||||
|
.sort((a, b) => a.mtimeMs - b.mtimeMs);
|
||||||
|
for (const e of evictable) {
|
||||||
|
if (remaining <= limit) break;
|
||||||
|
await rm(e.path, { force: true });
|
||||||
|
this.originals.delete(e.fileID);
|
||||||
|
remaining -= e.size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.originalsUsedBytes = remaining;
|
||||||
|
this.originalsLimitBytes = limit;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run limit work one at a time; failures are swallowed so a transient
|
||||||
|
// statfs or unlink error never rejects the read or write that triggered it.
|
||||||
|
private serializeEnforce(work: () => Promise<void>): Promise<void> {
|
||||||
|
const next = this.enforcing.then(work).catch(() => undefined);
|
||||||
|
this.enforcing = next;
|
||||||
|
return next;
|
||||||
|
}
|
||||||
|
|
||||||
private async ensureDir(dir: string): Promise<void> {
|
private async ensureDir(dir: string): Promise<void> {
|
||||||
// chmod after mkdir so the mode is tightened even when the directory
|
// chmod after mkdir so the mode is tightened even when the directory
|
||||||
// already existed with a looser one; mkdir alone would not.
|
// already existed with a looser one; mkdir alone would not.
|
||||||
|
|||||||
@@ -132,6 +132,14 @@ export interface LibraryOptions {
|
|||||||
// Overrides the client's own `contentSource()`; mainly for tests that drive
|
// Overrides the client's own `contentSource()`; mainly for tests that drive
|
||||||
// the cache with a stand-in source.
|
// the cache with a stand-in source.
|
||||||
contentSource?: ContentSource;
|
contentSource?: ContentSource;
|
||||||
|
// Bound on `cacheDirectory/originals` (default 100 GiB) and the free space
|
||||||
|
// to protect on its volume (default 50 GiB). The effective limit adapts
|
||||||
|
// down as the disk fills; `status().originalsLimitBytes` reports it.
|
||||||
|
cacheOriginalsMaxBytes?: number;
|
||||||
|
freeBelowBytes?: number;
|
||||||
|
// Whether an original is pinned and so never evicted (favorites + latest
|
||||||
|
// week; the precache unit #48 supplies the set).
|
||||||
|
isOriginalPinned?: (fileID: number) => boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LibraryStatus {
|
export interface LibraryStatus {
|
||||||
@@ -153,6 +161,10 @@ export interface LibraryStatus {
|
|||||||
// when ML fetching is disabled.
|
// when ML fetching is disabled.
|
||||||
mlStored?: number;
|
mlStored?: number;
|
||||||
mlIndexed?: number;
|
mlIndexed?: number;
|
||||||
|
// Bytes stored in the originals cache and the effective size limit as of the
|
||||||
|
// last write or open; undefined when no content cache is open.
|
||||||
|
originalsUsedBytes?: number;
|
||||||
|
originalsLimitBytes?: number;
|
||||||
closed: boolean;
|
closed: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -289,6 +301,9 @@ export class Library {
|
|||||||
cacheDirectory,
|
cacheDirectory,
|
||||||
downloadDirectory: opts.downloadDirectory,
|
downloadDirectory: opts.downloadDirectory,
|
||||||
getFile: (fileID) => store.getFileByID(fileID),
|
getFile: (fileID) => store.getFileByID(fileID),
|
||||||
|
cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes,
|
||||||
|
freeBelowBytes: opts.freeBelowBytes,
|
||||||
|
isPinned: opts.isOriginalPinned,
|
||||||
});
|
});
|
||||||
await cache.open();
|
await cache.open();
|
||||||
}
|
}
|
||||||
@@ -364,6 +379,7 @@ export class Library {
|
|||||||
files += this.store.listFiles(c.id).length;
|
files += this.store.listFiles(c.id).length;
|
||||||
}
|
}
|
||||||
const ml = this.mldata?.stats();
|
const ml = this.mldata?.stats();
|
||||||
|
const originals = this.cache?.originalsStatus();
|
||||||
return {
|
return {
|
||||||
userID: this.store.userID,
|
userID: this.store.userID,
|
||||||
collections: collections.length,
|
collections: collections.length,
|
||||||
@@ -374,6 +390,8 @@ export class Library {
|
|||||||
lastMLError: this.lastMLError,
|
lastMLError: this.lastMLError,
|
||||||
mlStored: ml?.stored,
|
mlStored: ml?.stored,
|
||||||
mlIndexed: ml?.indexed,
|
mlIndexed: ml?.indexed,
|
||||||
|
originalsUsedBytes: originals?.usedBytes,
|
||||||
|
originalsLimitBytes: originals?.limitBytes,
|
||||||
closed: this.closed,
|
closed: this.closed,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,295 @@
|
|||||||
|
/**
|
||||||
|
* Tests for the originals cache size limit and LRU eviction (issue #47),
|
||||||
|
* layered on the on-disk content cache (#46).
|
||||||
|
*
|
||||||
|
* Only `cacheDirectory/originals` is bounded and evicted. Before each original
|
||||||
|
* write the effective limit is
|
||||||
|
* min(cacheOriginalsMaxBytes, bytesUsedByOriginals + bytesFree - freeBelowBytes)
|
||||||
|
* with `bytesFree` read from `statfs` on the volume holding `cacheDirectory`.
|
||||||
|
* The limit therefore falls as the disk fills and rises as space returns. When
|
||||||
|
* a write would cross the limit, least-recently-used originals are removed until
|
||||||
|
* it fits; pinned files are skipped, and if only pinned files remain the write
|
||||||
|
* proceeds over-limit. Last-use is the file `mtime`, bumped whenever a read
|
||||||
|
* returns an original's path, so ordering survives a restart with no ledger.
|
||||||
|
*
|
||||||
|
* `statfs` is injected so the adaptive limit is exercised deterministically:
|
||||||
|
* `bsize` is 1, so `bavail` is the free byte count the formula sees. The source
|
||||||
|
* writes a controllable number of bytes per file, and tests set each stored
|
||||||
|
* file's `mtime` explicitly so LRU order does not depend on wall-clock timing.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync, existsSync, statSync, utimesSync } from "node:fs";
|
||||||
|
import { writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import {
|
||||||
|
ContentCache,
|
||||||
|
type ContentSource,
|
||||||
|
type StatFsFn,
|
||||||
|
} from "../../src/library/content.js";
|
||||||
|
import { RequestPools } from "../../src/library/pools.js";
|
||||||
|
import type { EnteFile } from "../../src/model/types.js";
|
||||||
|
|
||||||
|
const file = (id: number): EnteFile => ({
|
||||||
|
id,
|
||||||
|
collectionID: 1,
|
||||||
|
ownerID: 1,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
metadata: {
|
||||||
|
title: `file-${id}.jpg`,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: 0,
|
||||||
|
modificationTime: 0,
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: "aGVhZGVy" },
|
||||||
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
||||||
|
updationTime: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
// A source that writes a controllable number of bytes per file (default 10).
|
||||||
|
class SizedSource implements ContentSource {
|
||||||
|
sizeFor = new Map<number, number>();
|
||||||
|
|
||||||
|
async original(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
}): Promise<{ bytesWritten: number }> {
|
||||||
|
const n = this.sizeFor.get(args.file.id) ?? 10;
|
||||||
|
await writeFile(args.destination, Buffer.alloc(n, 1));
|
||||||
|
return { bytesWritten: n };
|
||||||
|
}
|
||||||
|
|
||||||
|
async thumbnail(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
}): Promise<{ bytesWritten: number }> {
|
||||||
|
await writeFile(args.destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let root: string;
|
||||||
|
let cacheDir: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-eviction-"));
|
||||||
|
cacheDir = join(root, "cache");
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
const originalPath = (id: number): string =>
|
||||||
|
join(cacheDir, "originals", `${id}.jpg`);
|
||||||
|
|
||||||
|
// Pin an original's mtime to a fixed second-resolution instant so LRU order is
|
||||||
|
// deterministic. Lower `seconds` = older = evicted first.
|
||||||
|
const setMtime = (id: number, seconds: number): void => {
|
||||||
|
utimesSync(originalPath(id), seconds, seconds);
|
||||||
|
};
|
||||||
|
|
||||||
|
const buildCache = (args: {
|
||||||
|
source?: ContentSource;
|
||||||
|
files?: EnteFile[];
|
||||||
|
statfs: StatFsFn;
|
||||||
|
cacheOriginalsMaxBytes?: number;
|
||||||
|
freeBelowBytes?: number;
|
||||||
|
isPinned?: (fileID: number) => boolean;
|
||||||
|
}): { cache: ContentCache; source: SizedSource } => {
|
||||||
|
const source = (args.source as SizedSource) ?? new SizedSource();
|
||||||
|
const byID = new Map<number, EnteFile>();
|
||||||
|
for (const f of args.files ?? [file(1), file(2), file(3), file(4), file(5)])
|
||||||
|
byID.set(f.id, f);
|
||||||
|
const cache = new ContentCache({
|
||||||
|
pools: new RequestPools(),
|
||||||
|
source,
|
||||||
|
cacheDirectory: cacheDir,
|
||||||
|
getFile: (id) => byID.get(id),
|
||||||
|
statfs: args.statfs,
|
||||||
|
cacheOriginalsMaxBytes: args.cacheOriginalsMaxBytes,
|
||||||
|
freeBelowBytes: args.freeBelowBytes,
|
||||||
|
isPinned: args.isPinned,
|
||||||
|
});
|
||||||
|
return { cache, source };
|
||||||
|
};
|
||||||
|
|
||||||
|
// Plenty of free space, so the configured max governs the limit unless a test
|
||||||
|
// dials it down. `bsize` of 1 makes `bavail` the free byte count.
|
||||||
|
const abundantFree: StatFsFn = async () => ({
|
||||||
|
bsize: 1,
|
||||||
|
bavail: 1_000_000_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("originals eviction", () => {
|
||||||
|
it("removes least-recently-used originals to fit the limit", async () => {
|
||||||
|
// Each original is 10 bytes; a 25-byte cap holds two.
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 25,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
await cache.original(1);
|
||||||
|
setMtime(1, 1000);
|
||||||
|
await cache.original(2);
|
||||||
|
setMtime(2, 2000);
|
||||||
|
// Writing the third (total 30 > 25) evicts the oldest, file 1.
|
||||||
|
await cache.original(3);
|
||||||
|
|
||||||
|
expect(existsSync(originalPath(1))).toBe(false);
|
||||||
|
expect(existsSync(originalPath(2))).toBe(true);
|
||||||
|
expect(existsSync(originalPath(3))).toBe(true);
|
||||||
|
expect(cache.pathsFor(1).originalPath).toBeUndefined();
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(20);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips pinned files, evicting the oldest unpinned instead", async () => {
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 25,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
isPinned: (id) => id === 1,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
await cache.original(1);
|
||||||
|
setMtime(1, 1000); // oldest, but pinned
|
||||||
|
await cache.original(2);
|
||||||
|
setMtime(2, 2000);
|
||||||
|
await cache.original(3);
|
||||||
|
|
||||||
|
// File 1 is oldest but pinned, so file 2 is evicted instead.
|
||||||
|
expect(existsSync(originalPath(1))).toBe(true);
|
||||||
|
expect(existsSync(originalPath(2))).toBe(false);
|
||||||
|
expect(existsSync(originalPath(3))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("proceeds over-limit when only pinned files remain", async () => {
|
||||||
|
// A 5-byte cap cannot hold even one 10-byte original.
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 5,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
isPinned: () => true,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const result = await cache.original(1);
|
||||||
|
|
||||||
|
expect(existsSync(result.path)).toBe(true);
|
||||||
|
const status = cache.originalsStatus();
|
||||||
|
expect(status.usedBytes).toBe(10);
|
||||||
|
expect(status.usedBytes).toBeGreaterThan(status.limitBytes ?? 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a just-written original larger than the limit, evicting it only on a later write", async () => {
|
||||||
|
// A 5-byte cap cannot hold even one 10-byte original, but a fetch must
|
||||||
|
// never evict the file it just wrote and is about to return.
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 5,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const result = await cache.original(1);
|
||||||
|
|
||||||
|
// The just-written original survives its own over-budget write and the
|
||||||
|
// returned path exists on disk.
|
||||||
|
expect(existsSync(result.path)).toBe(true);
|
||||||
|
expect(existsSync(originalPath(1))).toBe(true);
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(10);
|
||||||
|
setMtime(1, 1000);
|
||||||
|
|
||||||
|
// A later write finds file 1 eligible and evicts it to make room, while
|
||||||
|
// the newly written file 2 is itself kept over-limit.
|
||||||
|
await cache.original(2);
|
||||||
|
expect(existsSync(originalPath(1))).toBe(false);
|
||||||
|
expect(existsSync(originalPath(2))).toBe(true);
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(10);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("adapts the limit down as the disk fills and up as space returns", async () => {
|
||||||
|
// The configured max is generous; free space drives the limit. Eviction
|
||||||
|
// only kicks in once free space falls below the protected reserve.
|
||||||
|
let free = 1_000_000;
|
||||||
|
const statfs: StatFsFn = async () => ({ bsize: 1, bavail: free });
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs,
|
||||||
|
cacheOriginalsMaxBytes: 1000,
|
||||||
|
freeBelowBytes: 100,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
// Ample free space: the limit is the configured max and nothing is
|
||||||
|
// evicted as three 10-byte originals accumulate.
|
||||||
|
await cache.original(1);
|
||||||
|
setMtime(1, 1000);
|
||||||
|
await cache.original(2);
|
||||||
|
setMtime(2, 2000);
|
||||||
|
await cache.original(3);
|
||||||
|
setMtime(3, 3000);
|
||||||
|
expect(cache.originalsStatus().limitBytes).toBe(1000);
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(30);
|
||||||
|
|
||||||
|
// The disk fills: only 95 bytes free, below the 100-byte reserve. The
|
||||||
|
// next write (used 40) sees limit = min(1000, 40 + 95 - 100) = 35 and
|
||||||
|
// evicts the oldest original (file 1) to fit.
|
||||||
|
free = 95;
|
||||||
|
await cache.original(4);
|
||||||
|
expect(cache.originalsStatus().limitBytes).toBe(35);
|
||||||
|
expect(existsSync(originalPath(1))).toBe(false);
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(30);
|
||||||
|
setMtime(4, 4000);
|
||||||
|
|
||||||
|
// Space returns: the limit rises back to the configured max and the
|
||||||
|
// next write is kept without eviction.
|
||||||
|
free = 1_000_000;
|
||||||
|
await cache.original(5);
|
||||||
|
expect(cache.originalsStatus().limitBytes).toBe(1000);
|
||||||
|
expect(existsSync(originalPath(4))).toBe(true);
|
||||||
|
expect(existsSync(originalPath(5))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bumps an original's mtime when a read returns its path", async () => {
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 1000,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
await cache.original(1);
|
||||||
|
// Age the file well into the past.
|
||||||
|
setMtime(1, 1000);
|
||||||
|
expect(statSync(originalPath(1)).mtimeMs).toBeLessThan(2_000_000);
|
||||||
|
|
||||||
|
// A second read is a cache hit that must touch the file.
|
||||||
|
const before = Date.now();
|
||||||
|
await cache.original(1);
|
||||||
|
expect(statSync(originalPath(1)).mtimeMs).toBeGreaterThanOrEqual(
|
||||||
|
before - 2000,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never counts or evicts thumbnails", async () => {
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 5,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
await cache.thumbnail(1);
|
||||||
|
await cache.thumbnail(2);
|
||||||
|
|
||||||
|
expect(cache.pathsFor(1).thumbnailPath).toBeDefined();
|
||||||
|
expect(cache.pathsFor(2).thumbnailPath).toBeDefined();
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user