clawbot
f65c237e2c
check / check (push) Waiting to run
`quak get` and `quak get-thumb` used to wait for a refresh, which on an empty cache fetched the files of every collection. Now, on an empty cache, `freshFile` reads the collection list, then one collection's files at a time, and stops at the first that holds the file. That search stores nothing. Once the cache holds the account's collections, `freshFile` refreshes as before and looks the file up in the cache. The commands fetch the content through the new `Library.originalOf()` and `Library.thumbnailOf()`, which take the file record itself, since the store may not hold it. On an empty cache they no longer print a refresh's progress lines. The `-q` test that relied on `get` refreshing now runs `helper fix-missing-thumbnails`. Model: opus-5-5
1116 lines
44 KiB
TypeScript
1116 lines
44 KiB
TypeScript
// The on-disk content and thumbnail cache keyed by fileID (issue #46).
|
|
//
|
|
// Layout under `cacheDirectory`: `originals/<fileID>.<ext>` and
|
|
// `thumbnails/<fileID>.<ext>`, flat directories at 0700 with files at 0600. A
|
|
// live photo's original is two files, its image and its video, with
|
|
// `originals/<fileID>.livephoto.json` naming them. Content appears only by the
|
|
// streaming atomic writer's rename (the download layer, #40), so a file that
|
|
// exists is whole — "present means complete". The directory listing taken at
|
|
// `open()` is the record of what is cached, and the orphan temp files a crashed
|
|
// write may have left are reaped there.
|
|
//
|
|
// A fetch goes through the shared request pools (#45): the content pool for
|
|
// originals, the thumbnail pool for thumbnails. The pool limits concurrency,
|
|
// orders on-demand work ahead of background, and dedups by key so a fileID
|
|
// requested twice while the first is still in flight downloads once.
|
|
// `fetchOriginal` passes no key, so it is not shared with a fetch of the same
|
|
// file already in flight.
|
|
//
|
|
// Integrity. The reused streaming decrypt is the enforced guarantee: every
|
|
// chunk is authenticated and the writer renames the file into place only once
|
|
// the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and
|
|
// nothing is stored. For an original whose metadata records a content hash
|
|
// (`FileMetadata.hash`), the writer also hashes the decrypted bytes and stores
|
|
// nothing if they differ, failing the fetch with an error naming the file. An
|
|
// original with no recorded hash is stored unchecked, as the upstream client
|
|
// does; thumbnails have none. On top of that this module refuses to record a
|
|
// stored file that came out empty.
|
|
//
|
|
// It also names each original's save path under the download directory
|
|
// (`savePath`), where `Photo.download()` and `lib.backup()` put it. A copy in
|
|
// the cache does not count as saved there, but is copied there rather than
|
|
// fetched again. An original stored there whose bytes do not match its content
|
|
// hash is replaced only by a download that does (`replaceOriginal`).
|
|
|
|
import { createReadStream, existsSync, readFileSync, statSync } from "node:fs";
|
|
import {
|
|
chmod,
|
|
copyFile,
|
|
mkdir,
|
|
readdir,
|
|
rename,
|
|
rm,
|
|
stat,
|
|
statfs,
|
|
utimes,
|
|
} from "node:fs/promises";
|
|
import { basename, dirname, extname, join } from "node:path";
|
|
|
|
import type { ApiClient } from "../api/client.js";
|
|
import {
|
|
chunkHashFinal,
|
|
chunkHashInit,
|
|
chunkHashUpdate,
|
|
init,
|
|
} from "../crypto/index.js";
|
|
import {
|
|
downloadFile,
|
|
downloadThumbnail,
|
|
fsyncPath,
|
|
type ProgressCallback,
|
|
removeLeftoverTempFiles,
|
|
writeAtomic,
|
|
} from "../download/index.js";
|
|
import { safeExtension, withExtension } from "../filename.js";
|
|
import type { EnteFile } from "../model/types.js";
|
|
import type { Priority, RequestPools } from "./pools.js";
|
|
import { takenAtOf } from "./records.js";
|
|
|
|
const DIR_MODE = 0o700;
|
|
const FILE_MODE = 0o600;
|
|
const GIB = 1024 * 1024 * 1024;
|
|
// Owner ruling (#36): bound the originals cache at 100 GiB, but back off when
|
|
// the volume has under 50 GiB free so the cache never crowds the disk.
|
|
export const DEFAULT_ORIGINALS_MAX_BYTES = 100 * GIB;
|
|
export const DEFAULT_FREE_BELOW_BYTES = 50 * GIB;
|
|
// Ente thumbnails are always JPEG, so the cache stores them with a fixed
|
|
// extension rather than deriving one from the (image or video) title.
|
|
const THUMBNAIL_EXT = ".jpg";
|
|
|
|
type Kind = "original" | "thumbnail";
|
|
|
|
// An original write in progress, with the IDs of the concurrent original writes
|
|
// it overlaps (recorded both ways as writes begin, cleared when the write ends).
|
|
interface OriginalWrite {
|
|
fileID: number;
|
|
overlaps: Set<number>;
|
|
}
|
|
|
|
// The priority a caller attaches to a thumbnail prefetch. The pool has two
|
|
// tiers, so this three-value surface collapses onto them: only a currently
|
|
// visible thumbnail preempts (on-demand); "ahead" prefetch and speculative
|
|
// "background" work both yield to it.
|
|
export type ThumbnailPriority = "visible" | "ahead" | "background";
|
|
|
|
const poolPriorityOf = (priority: ThumbnailPriority): Priority =>
|
|
priority === "visible" ? "on-demand" : "background";
|
|
|
|
export interface ContentResult {
|
|
path: string;
|
|
bytes: number;
|
|
// A live photo's video. `path` and `bytes` are then its image's.
|
|
videoPath?: string;
|
|
}
|
|
|
|
// Progress for a single `original`/`thumbnail` call. A present file emits one
|
|
// `skipped` event and nothing else; a fetched file emits `downloading` as
|
|
// plaintext lands and a final `done`.
|
|
export type ContentEvent =
|
|
| { status: "skipped"; bytes: number }
|
|
| { status: "downloading"; bytesDone: number }
|
|
| { status: "done"; bytes: number };
|
|
|
|
export interface ContentOptions {
|
|
onProgress?: (event: ContentEvent) => void;
|
|
}
|
|
|
|
// The Photo-facing content surface (the read wrappers call these). The cache
|
|
// implements it; a library opened without a content source leaves it absent.
|
|
export interface PhotoContent {
|
|
original(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
|
|
thumbnail(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
|
|
// Put the original at the save path of `file`, the copy the `Photo` holds,
|
|
// and return it there.
|
|
download(file: EnteFile): Promise<ContentResult>;
|
|
}
|
|
|
|
export interface EnsureResult {
|
|
fileID: number;
|
|
path?: string;
|
|
error?: string;
|
|
}
|
|
|
|
export interface EnsureEvent {
|
|
fileID: number;
|
|
status: "skipped" | "done" | "failed" | "aborted";
|
|
path?: string;
|
|
error?: string;
|
|
}
|
|
|
|
export interface EnsureOptions {
|
|
fileIDs: number[];
|
|
priority: ThumbnailPriority;
|
|
signal?: AbortSignal;
|
|
onProgress?: (event: EnsureEvent) => void;
|
|
}
|
|
|
|
export interface ThumbnailsAPI {
|
|
ensure(args: EnsureOptions): Promise<EnsureResult[]>;
|
|
}
|
|
|
|
// The byte source the cache fetches through. The real implementation streams
|
|
// and decrypts to the destination via the download layer; tests inject a
|
|
// stand-in so the cache logic runs with no crypto and no network. Pool routing,
|
|
// dedup, present-checks and integrity live in the cache, not here.
|
|
export interface ContentSource {
|
|
// Writes the original at `destination`. A live photo is written beside it
|
|
// as its image and its video instead, and their paths are returned, as
|
|
// `downloadFile` does.
|
|
original(args: {
|
|
file: EnteFile;
|
|
destination: string;
|
|
onProgress?: ProgressCallback;
|
|
}): Promise<{ bytesWritten: number; path?: string; videoPath?: string }>;
|
|
thumbnail(args: {
|
|
file: EnteFile;
|
|
destination: string;
|
|
onProgress?: ProgressCallback;
|
|
}): Promise<{ bytesWritten: number }>;
|
|
}
|
|
|
|
// The production source: each fetch is the download layer's request +
|
|
// streaming decrypt + atomic write + retry as one unit.
|
|
export const makeDownloadContentSource = (api: ApiClient): ContentSource => ({
|
|
original: ({ file, destination, onProgress }) =>
|
|
downloadFile(api, file, destination, onProgress),
|
|
thumbnail: ({ file, destination, onProgress }) =>
|
|
downloadThumbnail(api, file, destination, onProgress),
|
|
});
|
|
|
|
export interface CachedPaths {
|
|
originalPath?: string;
|
|
thumbnailPath?: string;
|
|
}
|
|
|
|
// The slice of `fs.statfs` the eviction limit needs: `bavail` is the blocks
|
|
// available to an unprivileged writer and `bsize` their size, so
|
|
// `bavail * bsize` is the free byte count. Injectable so tests drive the
|
|
// adaptive limit without a real volume.
|
|
export interface StatFsResult {
|
|
bsize: number;
|
|
bavail: number;
|
|
}
|
|
export type StatFsFn = (path: string) => Promise<StatFsResult>;
|
|
|
|
const realStatFs: StatFsFn = async (path) => {
|
|
const s = await statfs(path);
|
|
return { bsize: s.bsize, bavail: s.bavail };
|
|
};
|
|
|
|
// The current usage and effective limit of the originals cache, in bytes.
|
|
// `limitBytes` is the adaptive ceiling last computed (see `originalsLimit`).
|
|
export interface OriginalsStatus {
|
|
usedBytes: number;
|
|
limitBytes?: number;
|
|
}
|
|
|
|
export interface ContentCacheOptions {
|
|
pools: RequestPools;
|
|
source: ContentSource;
|
|
cacheDirectory: string;
|
|
// The root of the save paths. An original already stored at its save path
|
|
// counts as present, so the cache serves it rather than fetching a second
|
|
// copy.
|
|
downloadDirectory: string;
|
|
// Resolve any membership of a file; every membership shares the underlying
|
|
// content key, so any one decrypts the same bytes.
|
|
getFile: (fileID: number) => EnteFile | undefined;
|
|
// Hard ceiling on `cacheDirectory/originals` (default 100 GiB) and the free
|
|
// space to protect on the volume (default 50 GiB). The effective limit is
|
|
// the lesser of the ceiling and what fits above the protected free space.
|
|
cacheOriginalsMaxBytes?: number;
|
|
freeBelowBytes?: number;
|
|
// Whether an original is pinned (favorites + latest week; the precache unit
|
|
// #48 supplies the set). Pinned originals are never evicted; when only
|
|
// pinned originals remain the cache runs over-limit until the set shrinks.
|
|
isPinned?: (fileID: number) => boolean;
|
|
// Free-space probe on the volume holding `cacheDirectory`; defaults to the
|
|
// real `fs.statfs`.
|
|
statfs?: StatFsFn;
|
|
}
|
|
|
|
// Thrown inside a pooled task to drop a queued fetch that was aborted before it
|
|
// started running. Never escapes `ensureThumbnails`.
|
|
class AbortDrop extends Error {
|
|
constructor() {
|
|
super("aborted");
|
|
this.name = "AbortDrop";
|
|
}
|
|
}
|
|
|
|
// The name of a file's original in the cache's originals/: `<fileID><ext>`, the
|
|
// extension taken from the title (or `.bin`).
|
|
export const nameInOriginals = (file: EnteFile): string =>
|
|
`${file.id}${safeExtension(file.metadata.title)}`;
|
|
|
|
const pad = (n: number): string => String(n).padStart(2, "0");
|
|
|
|
// Where the original of `file` is saved under `root`:
|
|
// `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID><ext>`, dated by the photo's
|
|
// `takenAt` in this machine's time zone, with the extension taken from the
|
|
// title (or `.bin`). A live photo is stored as its image and its video beside
|
|
// this path, each with the extension found inside the live photo.
|
|
export const savePath = (root: string, file: EnteFile): string => {
|
|
const taken = new Date(takenAtOf(file));
|
|
const year = String(taken.getFullYear());
|
|
const month = `${year}-${pad(taken.getMonth() + 1)}`;
|
|
const day = `${month}-${pad(taken.getDate())}`;
|
|
const ext = safeExtension(file.metadata.title);
|
|
return join(root, year, month, day, `${day}.${file.id}${ext}`);
|
|
};
|
|
|
|
// The fileID a cache filename encodes, or undefined when the name is not one
|
|
// the cache writes (`<digits><ext>`).
|
|
const fileIDFromName = (name: string): number | undefined => {
|
|
const base = name.slice(0, name.length - extname(name).length);
|
|
if (!/^\d+$/.test(base)) return undefined;
|
|
const id = Number(base);
|
|
return Number.isSafeInteger(id) ? id : undefined;
|
|
};
|
|
|
|
// Size of a regular file, or undefined if it is absent (or not a regular file).
|
|
const fileSize = (path: string): number | undefined => {
|
|
try {
|
|
const s = statSync(path);
|
|
return s.isFile() ? s.size : undefined;
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
};
|
|
|
|
// Whether `path` is a regular file with content. A zero-byte file is the shape
|
|
// an aborted write leaves, so it does not count.
|
|
const hasContent = (path: string | undefined): boolean =>
|
|
path !== undefined && (fileSize(path) ?? 0) > 0;
|
|
|
|
// A live photo's image and video are named with the extensions from inside its
|
|
// ZIP, so their names alone do not say which is which. Wherever the cache or a
|
|
// save path stores one, a JSON file of this name beside them names both.
|
|
// `name` is the original's name without its extension: `<fileID>` in the
|
|
// cache, `YYYY-MM-DD.<fileID>` at the save path.
|
|
const livePhotoJSONName = (name: string): string => `${name}.livephoto.json`;
|
|
|
|
// The image and video that the live photo's JSON file in `dir` names, or
|
|
// undefined when there is none. Only `name` with an extension is taken, so the
|
|
// file cannot point outside `dir`.
|
|
const readLivePhotoJSON = (
|
|
dir: string,
|
|
name: string,
|
|
): { path: string; videoPath: string } | undefined => {
|
|
const valid = (part: unknown): part is string =>
|
|
typeof part === "string" && part === `${name}${safeExtension(part)}`;
|
|
try {
|
|
const { image, video } = JSON.parse(
|
|
readFileSync(join(dir, livePhotoJSONName(name)), "utf-8"),
|
|
);
|
|
if (valid(image) && valid(video)) {
|
|
return { path: join(dir, image), videoPath: join(dir, video) };
|
|
}
|
|
} catch {
|
|
// No such file, or not one quak wrote.
|
|
}
|
|
return undefined;
|
|
};
|
|
|
|
// Write the JSON file naming a live photo's image and video, both in `dir`.
|
|
export const writeLivePhotoJSON = (
|
|
dir: string,
|
|
name: string,
|
|
stored: { path: string; videoPath: string },
|
|
): Promise<void> =>
|
|
writeAtomic(
|
|
join(dir, livePhotoJSONName(name)),
|
|
new TextEncoder().encode(
|
|
JSON.stringify({
|
|
image: basename(stored.path),
|
|
video: basename(stored.videoPath),
|
|
}),
|
|
),
|
|
);
|
|
|
|
// The original of `file` as stored in `dir` under `name` (without its
|
|
// extension), when all of it is there: `<name><ext>`, or a live photo's image
|
|
// and video.
|
|
export const storedOriginal = (
|
|
dir: string,
|
|
name: string,
|
|
file: EnteFile,
|
|
): { path: string; videoPath?: string } | undefined => {
|
|
if (file.metadata.fileType !== "livePhoto") {
|
|
const path = join(dir, `${name}${safeExtension(file.metadata.title)}`);
|
|
return hasContent(path) ? { path } : undefined;
|
|
}
|
|
const stored = readLivePhotoJSON(dir, name);
|
|
return stored !== undefined &&
|
|
hasContent(stored.path) &&
|
|
hasContent(stored.videoPath)
|
|
? stored
|
|
: undefined;
|
|
};
|
|
|
|
// The original of `file` as stored at its save path under `root`, when all of
|
|
// it is there.
|
|
export const storedAtSavePath = (
|
|
root: string,
|
|
file: EnteFile,
|
|
): { path: string; videoPath?: string } | undefined => {
|
|
const path = savePath(root, file);
|
|
return storedOriginal(dirname(path), basename(path, extname(path)), file);
|
|
};
|
|
|
|
// The content hash of the original stored at `stored`, computed as the download
|
|
// check computes it: over each file's bytes, read in chunks, and for a live
|
|
// photo `<imageHash>:<videoHash>`.
|
|
export const storedHash = async (stored: {
|
|
path: string;
|
|
videoPath?: string;
|
|
}): Promise<string> => {
|
|
await init();
|
|
const hashFile = async (path: string): Promise<string> => {
|
|
const state = chunkHashInit();
|
|
for await (const chunk of createReadStream(path)) {
|
|
chunkHashUpdate(state, chunk as Buffer);
|
|
}
|
|
return chunkHashFinal(state);
|
|
};
|
|
const hash = await hashFile(stored.path);
|
|
if (stored.videoPath === undefined) return hash;
|
|
return `${hash}:${await hashFile(stored.videoPath)}`;
|
|
};
|
|
|
|
// Copy bytes into `dest` via a temp file in the same directory plus rename, so
|
|
// `dest` appears only once it is whole ("present means complete"). As in the
|
|
// download writer, the temp file is fsynced before the rename and the directory
|
|
// after it, so a power cut cannot leave a correctly named but short original.
|
|
// The temp name carries this process's ID so a later run can tell a leftover
|
|
// from a copy still in progress (see `removeLeftoverTempFiles`).
|
|
export const copyAtomic = async (src: string, dest: string): Promise<void> => {
|
|
if (src === dest) return;
|
|
const tmp = join(
|
|
dirname(dest),
|
|
`.quak-backup-${basename(dest)}-${process.pid}-${Math.random()
|
|
.toString(36)
|
|
.slice(2)}.tmp`,
|
|
);
|
|
try {
|
|
await copyFile(src, tmp);
|
|
await fsyncPath(tmp);
|
|
// `rename` replaces the destination's directory entry: an existing
|
|
// symlink at `dest` is replaced, not followed, and the new file has
|
|
// the temp file's permissions (copied from `src`).
|
|
await rename(tmp, dest);
|
|
await fsyncPath(dirname(dest));
|
|
} finally {
|
|
await rm(tmp, { force: true });
|
|
}
|
|
};
|
|
|
|
// Put the original of `file` at its save path under `root`, creating its
|
|
// folders. `get` is given the save path and returns where the original is: a
|
|
// fetch writes it there, and a copy the cache holds is copied there. A live
|
|
// photo's image and video go beside the save path, each with its own
|
|
// extension: when they came from the cache they are copied. Then the JSON file
|
|
// naming them is written, which is what makes the live photo count as stored.
|
|
export const placeOriginal = async (
|
|
root: string,
|
|
file: EnteFile,
|
|
get: (dest: string) => Promise<{ path: string; videoPath?: string }>,
|
|
): Promise<{ path: string; videoPath?: string }> => {
|
|
const dest = savePath(root, file);
|
|
await mkdir(dirname(dest), { recursive: true });
|
|
const got = await get(dest);
|
|
if (got.videoPath === undefined) {
|
|
await copyAtomic(got.path, dest);
|
|
return { path: dest };
|
|
}
|
|
const path = withExtension(dest, extname(got.path));
|
|
const videoPath = withExtension(dest, extname(got.videoPath));
|
|
if (got.path !== path) {
|
|
await copyAtomic(got.path, path);
|
|
await copyAtomic(got.videoPath, videoPath);
|
|
}
|
|
const name = basename(dest, extname(dest));
|
|
await writeLivePhotoJSON(dirname(dest), name, { path, videoPath });
|
|
return { path, videoPath };
|
|
};
|
|
|
|
// Replace the original of `file` stored at its save path under `root` with a
|
|
// new copy, keeping the stored one until the copy is complete and its hash,
|
|
// computed as `storedHash` computes it, is the content hash the file's metadata
|
|
// records. `get` is given a temporary file beside the save path and writes the
|
|
// copy there, a live photo as its image and its video beside that file, each
|
|
// with its own extension. Only a copy that matches is renamed over the stored
|
|
// one, and a live photo's image and video are both hashed before either is
|
|
// renamed. The image is renamed first: should the run stop, or the video's
|
|
// rename fail, before the video is renamed, the new image is left beside the
|
|
// old video, and a later backup with `verify` replaces the video if the video
|
|
// was the part that did not match. When `get` fails or the copy does not match,
|
|
// the temporary files are removed and the stored original stays as it was.
|
|
export const replaceOriginal = async (
|
|
root: string,
|
|
file: EnteFile,
|
|
get: (dest: string) => Promise<{ path: string; videoPath?: string }>,
|
|
): Promise<{ path: string; videoPath?: string }> => {
|
|
const dest = savePath(root, file);
|
|
const dir = dirname(dest);
|
|
// Named as the download writer names its temporary files; a live photo's
|
|
// image and video get this name with their own extensions. The next
|
|
// backup removes any that a killed run leaves (`removeLeftoverTempFiles`).
|
|
const tmp = join(
|
|
dir,
|
|
`.quak-${process.pid}-${Math.random().toString(36).slice(2)}.tmp`,
|
|
);
|
|
let got: { path: string; videoPath?: string } | undefined;
|
|
try {
|
|
got = await get(tmp);
|
|
if ((await storedHash(got)) !== file.metadata.hash) {
|
|
throw new Error(
|
|
"the downloaded original does not match its content hash either",
|
|
);
|
|
}
|
|
if (got.videoPath === undefined) {
|
|
await rename(got.path, dest);
|
|
await fsyncPath(dir);
|
|
return { path: dest };
|
|
}
|
|
const path = withExtension(dest, extname(got.path));
|
|
const videoPath = withExtension(dest, extname(got.videoPath));
|
|
await rename(got.path, path);
|
|
await rename(got.videoPath, videoPath);
|
|
const name = basename(dest, extname(dest));
|
|
await writeLivePhotoJSON(dir, name, { path, videoPath });
|
|
return { path, videoPath };
|
|
} finally {
|
|
// Whatever was not renamed into place. A failure to remove it must
|
|
// not replace the error that explains what went wrong.
|
|
for (const path of [tmp, got?.path, got?.videoPath]) {
|
|
if (path === undefined) continue;
|
|
await rm(path, { force: true }).catch(() => undefined);
|
|
}
|
|
}
|
|
};
|
|
|
|
export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|
private readonly pools: RequestPools;
|
|
private readonly source: ContentSource;
|
|
private readonly downloadDirectory: string;
|
|
private readonly getFile: (fileID: number) => EnteFile | undefined;
|
|
private readonly originalsDir: string;
|
|
private readonly thumbnailsDir: string;
|
|
// fileID -> absolute path of the cached bytes, and for a live photo's
|
|
// original its video's, seeded from the directory listing at open() and
|
|
// extended as fetches store new files.
|
|
private readonly originals = new Map<
|
|
number,
|
|
{ path: string; videoPath?: string }
|
|
>();
|
|
private readonly thumbnails = new Map<
|
|
number,
|
|
{ path: string; videoPath?: string }
|
|
>();
|
|
private readonly maxOriginalsBytes: number;
|
|
private readonly freeBelowBytes: number;
|
|
private readonly isPinned: (fileID: number) => boolean;
|
|
private readonly statfs: StatFsFn;
|
|
// The last measured usage and effective limit, refreshed at open() and after
|
|
// every original write; exposed through `originalsStatus`.
|
|
private originalsUsedBytes = 0;
|
|
private originalsLimitBytes?: number;
|
|
// Serializes limit enforcement so concurrent original writes never race on
|
|
// the map or delete each other's just-freed room.
|
|
private enforcing: Promise<void> = Promise.resolve();
|
|
// Original writes in progress. Writes for different files run concurrently
|
|
// (the content pool), so an eviction pass must never delete a file whose
|
|
// fetch has not yet returned. Each entry records the IDs of the concurrent
|
|
// original writes it overlaps — noted both ways as writes begin — and a
|
|
// write's eviction pass spares them all. Bounded by the pool's concurrency,
|
|
// so eviction is never deferred beyond the active working set.
|
|
private readonly inFlightOriginals = new Set<OriginalWrite>();
|
|
|
|
constructor(opts: ContentCacheOptions) {
|
|
this.pools = opts.pools;
|
|
this.source = opts.source;
|
|
this.downloadDirectory = opts.downloadDirectory;
|
|
this.getFile = opts.getFile;
|
|
this.originalsDir = join(opts.cacheDirectory, "originals");
|
|
this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails");
|
|
this.maxOriginalsBytes =
|
|
opts.cacheOriginalsMaxBytes ?? DEFAULT_ORIGINALS_MAX_BYTES;
|
|
this.freeBelowBytes = opts.freeBelowBytes ?? DEFAULT_FREE_BELOW_BYTES;
|
|
this.isPinned = opts.isPinned ?? (() => false);
|
|
this.statfs = opts.statfs ?? realStatFs;
|
|
}
|
|
|
|
// Prepare the cache directories, reap orphan temp files, and take the
|
|
// record of what is already cached. Called once before the cache serves.
|
|
// `isLivePhoto` says which files are live photos, whose original is only
|
|
// the image and video their JSON file names.
|
|
async open(
|
|
isLivePhoto: (fileID: number) => boolean = () => false,
|
|
): Promise<void> {
|
|
await this.ensureDir(this.originalsDir);
|
|
await this.ensureDir(this.thumbnailsDir);
|
|
await this.scan(this.originalsDir, this.originals, isLivePhoto);
|
|
await this.scan(this.thumbnailsDir, this.thumbnails, () => false);
|
|
// Publish the current usage and limit without evicting; a restart
|
|
// reuses whatever survived on disk. Eviction only ever fires on a write.
|
|
await this.refreshOriginalsLimit();
|
|
}
|
|
|
|
// The current originals usage and effective limit, both in bytes, as of the
|
|
// last write or open. `status().originalsLimitBytes` surfaces this.
|
|
originalsStatus(): OriginalsStatus {
|
|
return {
|
|
usedBytes: this.originalsUsedBytes,
|
|
limitBytes: this.originalsLimitBytes,
|
|
};
|
|
}
|
|
|
|
// The cache paths known for a file, for the record projection to expose as
|
|
// `originalPath`/`thumbnailPath`.
|
|
pathsFor(fileID: number): CachedPaths {
|
|
const out: CachedPaths = {};
|
|
const original = this.originals.get(fileID);
|
|
if (original !== undefined) out.originalPath = original.path;
|
|
const thumbnail = this.thumbnails.get(fileID);
|
|
if (thumbnail !== undefined) out.thumbnailPath = thumbnail.path;
|
|
return out;
|
|
}
|
|
|
|
async original(
|
|
fileID: number,
|
|
opts?: ContentOptions,
|
|
): Promise<ContentResult> {
|
|
const file = this.fileOrThrow(fileID);
|
|
return this.get(file, "original", "on-demand", opts?.onProgress);
|
|
}
|
|
|
|
async thumbnail(
|
|
fileID: number,
|
|
opts?: ContentOptions,
|
|
): Promise<ContentResult> {
|
|
const file = this.fileOrThrow(fileID);
|
|
return this.get(file, "thumbnail", "on-demand", opts?.onProgress);
|
|
}
|
|
|
|
// The original of `file`, fetched and cached as `original` does. `file` is
|
|
// used as given, so `getFile` need not know it.
|
|
async originalOf(
|
|
file: EnteFile,
|
|
opts?: ContentOptions,
|
|
): Promise<ContentResult> {
|
|
return this.get(file, "original", "on-demand", opts?.onProgress);
|
|
}
|
|
|
|
// The thumbnail of `file`, fetched and cached as `thumbnail` does. `file`
|
|
// is used as given, so `getFile` need not know it.
|
|
async thumbnailOf(
|
|
file: EnteFile,
|
|
opts?: ContentOptions,
|
|
): Promise<ContentResult> {
|
|
return this.get(file, "thumbnail", "on-demand", opts?.onProgress);
|
|
}
|
|
|
|
// Put the original at the save path of `file` under the download directory
|
|
// and return it there. `file` is the copy the `Photo` holds, so the path is
|
|
// the one its `savePath` names, even after a refresh changed the date. One
|
|
// already stored there is returned as it is; one the cache holds is copied
|
|
// from it; any other is fetched straight to the save path, with no copy
|
|
// left in the cache.
|
|
async download(file: EnteFile): Promise<ContentResult> {
|
|
const root = this.downloadDirectory;
|
|
const saved =
|
|
storedAtSavePath(root, file) ??
|
|
(await placeOriginal(root, file, (dest) =>
|
|
this.backupOriginal(file.id, dest),
|
|
));
|
|
return { ...saved, bytes: fileSize(saved.path) ?? 0 };
|
|
}
|
|
|
|
// Get an original for a save path. One not present anywhere is written
|
|
// straight to `destination` and recorded there, so no second copy lands
|
|
// in the cache; one already present is returned where it is.
|
|
async backupOriginal(
|
|
fileID: number,
|
|
destination: string,
|
|
): Promise<ContentResult> {
|
|
const result = await this.acquire(
|
|
this.fileOrThrow(fileID),
|
|
"original",
|
|
"on-demand",
|
|
undefined,
|
|
{ destination },
|
|
);
|
|
return {
|
|
path: result.path,
|
|
bytes: result.bytes,
|
|
videoPath: result.videoPath,
|
|
};
|
|
}
|
|
|
|
// Fetch an original through the content pool to `destination`, a live
|
|
// photo as its image and its video beside it, and return where it landed.
|
|
// Unlike `backupOriginal`, it fetches even when the cache or the save path
|
|
// already holds the original, and it records nothing: what it writes is
|
|
// the caller's. It is not shared with a fetch of the same file already
|
|
// under way, which writes somewhere else.
|
|
async fetchOriginal(
|
|
fileID: number,
|
|
destination: string,
|
|
): Promise<{ path: string; videoPath?: string }> {
|
|
const file = this.fileOrThrow(fileID);
|
|
return this.pools.content.run(
|
|
async () => {
|
|
const fetched = await this.fetchInto(
|
|
file,
|
|
destination,
|
|
"original",
|
|
undefined,
|
|
);
|
|
for (const path of [fetched.path, fetched.videoPath]) {
|
|
if (path !== undefined) await chmod(path, FILE_MODE);
|
|
}
|
|
return fetched;
|
|
},
|
|
{ priority: "on-demand" },
|
|
);
|
|
}
|
|
|
|
async ensure(args: EnsureOptions): Promise<EnsureResult[]> {
|
|
return this.ensureThumbnails(args);
|
|
}
|
|
|
|
async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> {
|
|
return this.ensureMany(
|
|
"thumbnail",
|
|
poolPriorityOf(args.priority),
|
|
args.fileIDs,
|
|
args.signal,
|
|
args.onProgress,
|
|
);
|
|
}
|
|
|
|
// Fill originals through the content pool for the precache (#48), always at
|
|
// background priority so an on-demand `original()` preempts the fill. A
|
|
// present original is a map lookup and no fetch; a per-file failure is
|
|
// returned, not thrown, so one bad file never halts a background sweep.
|
|
async ensureOriginals(args: {
|
|
fileIDs: number[];
|
|
signal?: AbortSignal;
|
|
onProgress?: (event: EnsureEvent) => void;
|
|
}): Promise<EnsureResult[]> {
|
|
return this.ensureMany(
|
|
"original",
|
|
"background",
|
|
args.fileIDs,
|
|
args.signal,
|
|
args.onProgress,
|
|
);
|
|
}
|
|
|
|
private async ensureMany(
|
|
kind: Kind,
|
|
priority: Priority,
|
|
fileIDs: number[],
|
|
signal: AbortSignal | undefined,
|
|
onProgress: ((event: EnsureEvent) => void) | undefined,
|
|
): Promise<EnsureResult[]> {
|
|
// Dedup the request list so a repeated fileID is fetched once and
|
|
// reported once, in first-requested order.
|
|
const seen = new Set<number>();
|
|
const unique: number[] = [];
|
|
for (const id of fileIDs) {
|
|
if (!seen.has(id)) {
|
|
seen.add(id);
|
|
unique.push(id);
|
|
}
|
|
}
|
|
return Promise.all(
|
|
unique.map((fileID) =>
|
|
this.ensureOne(fileID, kind, priority, signal, onProgress),
|
|
),
|
|
);
|
|
}
|
|
|
|
private async ensureOne(
|
|
fileID: number,
|
|
kind: Kind,
|
|
priority: Priority,
|
|
signal: AbortSignal | undefined,
|
|
onProgress: ((event: EnsureEvent) => void) | undefined,
|
|
): Promise<EnsureResult> {
|
|
try {
|
|
const file = this.fileOrThrow(fileID);
|
|
const result = await this.acquire(file, kind, priority, signal);
|
|
const status = result.cached ? "skipped" : "done";
|
|
onProgress?.({ fileID, status, path: result.path });
|
|
return { fileID, path: result.path };
|
|
} catch (err) {
|
|
if (err instanceof AbortDrop) {
|
|
onProgress?.({ fileID, status: "aborted" });
|
|
return { fileID, error: "aborted" };
|
|
}
|
|
const error = err instanceof Error ? err.message : String(err);
|
|
onProgress?.({ fileID, status: "failed", error });
|
|
return { fileID, error };
|
|
}
|
|
}
|
|
|
|
// The record `getFile` gives for `fileID`. It throws for a file `getFile`
|
|
// does not know.
|
|
private fileOrThrow(fileID: number): EnteFile {
|
|
const file = this.getFile(fileID);
|
|
if (!file) throw new Error(`content cache: unknown file ${fileID}`);
|
|
return file;
|
|
}
|
|
|
|
private async get(
|
|
file: EnteFile,
|
|
kind: Kind,
|
|
priority: Priority,
|
|
onProgress: ((event: ContentEvent) => void) | undefined,
|
|
): Promise<ContentResult> {
|
|
const onByte: ProgressCallback | undefined = onProgress
|
|
? (bytesDone) => onProgress({ status: "downloading", bytesDone })
|
|
: undefined;
|
|
const result = await this.acquire(file, kind, priority, undefined, {
|
|
onByte,
|
|
});
|
|
onProgress?.(
|
|
result.cached
|
|
? { status: "skipped", bytes: result.bytes }
|
|
: { status: "done", bytes: result.bytes },
|
|
);
|
|
return {
|
|
path: result.path,
|
|
bytes: result.bytes,
|
|
videoPath: result.videoPath,
|
|
};
|
|
}
|
|
|
|
// The core: return the cached path if present, else fetch through the pool,
|
|
// store, and return it. `cached` distinguishes a present hit (no network,
|
|
// no download event) from a fresh fetch. A fetched original is stored at
|
|
// `opts.destination` when given, instead of in `originalsDir`. A live
|
|
// photo's original is present only with its video, and is returned with
|
|
// it.
|
|
private async acquire(
|
|
file: EnteFile,
|
|
kind: Kind,
|
|
priority: Priority,
|
|
signal: AbortSignal | undefined,
|
|
opts?: { onByte?: ProgressCallback; destination?: string },
|
|
): Promise<{
|
|
path: string;
|
|
bytes: number;
|
|
videoPath?: string;
|
|
cached: boolean;
|
|
}> {
|
|
const fileID = file.id;
|
|
const isLivePhoto =
|
|
kind === "original" && file.metadata.fileType === "livePhoto";
|
|
|
|
const known = kind === "original" ? this.originals : this.thumbnails;
|
|
const cached = known.get(fileID);
|
|
if (cached !== undefined) {
|
|
const size = fileSize(cached.path);
|
|
if (
|
|
size !== undefined &&
|
|
size > 0 &&
|
|
(!isLivePhoto || hasContent(cached.videoPath))
|
|
) {
|
|
// Returning an original's path is a use: bump its mtime so LRU
|
|
// order reflects it and survives a restart with no ledger.
|
|
if (
|
|
kind === "original" &&
|
|
dirname(cached.path) === this.originalsDir
|
|
)
|
|
await this.touch(cached.path);
|
|
return { ...cached, bytes: size, cached: true };
|
|
}
|
|
// A recorded file that has since gone re-fetches below. So does a
|
|
// live photo recorded with no video: the cache opened before the
|
|
// library's records said it is a live photo, while its image and
|
|
// video had no JSON file beside them yet.
|
|
known.delete(fileID);
|
|
}
|
|
|
|
// An original already stored at its save path counts as present.
|
|
if (kind === "original") {
|
|
const stored = storedAtSavePath(this.downloadDirectory, file);
|
|
if (stored !== undefined) {
|
|
this.originals.set(fileID, stored);
|
|
return {
|
|
...stored,
|
|
bytes: fileSize(stored.path) ?? 0,
|
|
cached: true,
|
|
};
|
|
}
|
|
}
|
|
|
|
const dir =
|
|
kind === "original" ? this.originalsDir : this.thumbnailsDir;
|
|
const dest =
|
|
kind === "original"
|
|
? (opts?.destination ?? join(dir, nameInOriginals(file)))
|
|
: join(dir, `${fileID}${THUMBNAIL_EXT}`);
|
|
const pool =
|
|
kind === "original" ? this.pools.content : this.pools.thumbnails;
|
|
|
|
return pool.run(
|
|
async () => {
|
|
// Dropping queued work on abort: a task still waiting for a slot
|
|
// when the signal fired sees it here and never touches the
|
|
// network. A task already past this point is in flight and runs
|
|
// to completion.
|
|
if (signal?.aborted) throw new AbortDrop();
|
|
|
|
// Register this original among those in flight, linking it with
|
|
// every sibling already writing so neither evicts the other's
|
|
// file. Non-null iff this is an original.
|
|
const write =
|
|
kind === "original"
|
|
? this.beginOriginalWrite(fileID)
|
|
: null;
|
|
try {
|
|
const stored = await this.fetchInto(
|
|
file,
|
|
dest,
|
|
kind,
|
|
opts?.onByte,
|
|
);
|
|
for (const path of [stored.path, stored.videoPath]) {
|
|
if (path === undefined) continue;
|
|
await chmod(path, FILE_MODE);
|
|
if ((await stat(path)).size === 0) {
|
|
throw new Error(
|
|
`content cache: ${kind} ${fileID} stored empty`,
|
|
);
|
|
}
|
|
}
|
|
// `placeOriginal` records a live photo it saves.
|
|
if (
|
|
stored.videoPath !== undefined &&
|
|
opts?.destination === undefined
|
|
) {
|
|
await writeLivePhotoJSON(dir, String(fileID), {
|
|
path: stored.path,
|
|
videoPath: stored.videoPath,
|
|
});
|
|
}
|
|
known.set(fileID, stored);
|
|
// A fresh original may have crossed the limit; make room by
|
|
// evicting least-recently-used originals. An over-budget
|
|
// fetch keeps the file it returns, and no overlapping
|
|
// sibling is evicted. Thumbnails are never bounded.
|
|
if (write) await this.enforceOriginalsLimit(write);
|
|
const size = (await stat(stored.path)).size;
|
|
return { ...stored, bytes: size, cached: false };
|
|
} finally {
|
|
if (write) this.inFlightOriginals.delete(write);
|
|
}
|
|
},
|
|
{ priority, key: fileID },
|
|
);
|
|
}
|
|
|
|
// Fetch into `destination`, returning where the bytes landed: there, or
|
|
// for a live photo, its image and video beside it.
|
|
private async fetchInto(
|
|
file: EnteFile,
|
|
destination: string,
|
|
kind: Kind,
|
|
onProgress: ProgressCallback | undefined,
|
|
): Promise<{ path: string; videoPath?: string }> {
|
|
const args = { file, destination, onProgress };
|
|
if (kind === "thumbnail") {
|
|
await this.source.thumbnail(args);
|
|
return { path: destination };
|
|
}
|
|
const result = await this.source.original(args);
|
|
return {
|
|
path: result.path ?? destination,
|
|
videoPath: result.videoPath,
|
|
};
|
|
}
|
|
|
|
// Best-effort bump of a file's mtime to now; a failed touch must never fail
|
|
// the read it accompanies.
|
|
private async touch(path: string): Promise<void> {
|
|
const now = new Date();
|
|
await utimes(path, now, now).catch(() => undefined);
|
|
}
|
|
|
|
// Every stored original that lives under `originalsDir` (a save-path hit
|
|
// recorded in the map is excluded), with its size and mtime; a live
|
|
// photo's size includes its video. Entries whose file has vanished are
|
|
// dropped from the map. Save paths and thumbnails are never counted.
|
|
private async measureOriginals(): Promise<{
|
|
entries: {
|
|
fileID: number;
|
|
path: string;
|
|
videoPath?: string;
|
|
size: number;
|
|
mtimeMs: number;
|
|
}[];
|
|
used: number;
|
|
}> {
|
|
const entries: {
|
|
fileID: number;
|
|
path: string;
|
|
videoPath?: string;
|
|
size: number;
|
|
mtimeMs: number;
|
|
}[] = [];
|
|
let used = 0;
|
|
for (const [fileID, { path, videoPath }] of this.originals) {
|
|
if (dirname(path) !== this.originalsDir) continue;
|
|
try {
|
|
const s = await stat(path);
|
|
const size =
|
|
s.size +
|
|
(videoPath === undefined
|
|
? 0
|
|
: (await stat(videoPath)).size);
|
|
entries.push({
|
|
fileID,
|
|
path,
|
|
videoPath,
|
|
size,
|
|
mtimeMs: s.mtimeMs,
|
|
});
|
|
used += size;
|
|
} catch {
|
|
this.originals.delete(fileID);
|
|
}
|
|
}
|
|
return { entries, used };
|
|
}
|
|
|
|
// The effective ceiling on originals: the configured max, but no more than
|
|
// what fits once the protected free space is set aside. `used + free` is the
|
|
// volume space the cache could occupy; subtracting `freeBelowBytes` leaves
|
|
// the reserve untouched. Clamped at zero.
|
|
private async originalsLimit(used: number): Promise<number> {
|
|
const { bsize, bavail } = await this.statfs(this.originalsDir);
|
|
const free = bsize * bavail;
|
|
const adaptive = used + free - this.freeBelowBytes;
|
|
return Math.max(0, Math.min(this.maxOriginalsBytes, adaptive));
|
|
}
|
|
|
|
// Recompute and publish usage and limit without evicting (used at open()).
|
|
private refreshOriginalsLimit(): Promise<void> {
|
|
return this.serializeEnforce(async () => {
|
|
const { used } = await this.measureOriginals();
|
|
this.originalsUsedBytes = used;
|
|
this.originalsLimitBytes = await this.originalsLimit(used);
|
|
});
|
|
}
|
|
|
|
// Record a starting original write among those in flight, linking it with
|
|
// every sibling already writing so neither can evict the other's file.
|
|
private beginOriginalWrite(fileID: number): OriginalWrite {
|
|
const write: OriginalWrite = { fileID, overlaps: new Set() };
|
|
for (const other of this.inFlightOriginals) {
|
|
write.overlaps.add(other.fileID);
|
|
other.overlaps.add(fileID);
|
|
}
|
|
this.inFlightOriginals.add(write);
|
|
return write;
|
|
}
|
|
|
|
// Evict least-recently-used originals until usage fits the limit. Skipped:
|
|
// pinned originals, the file `write` just stored, and every original whose
|
|
// write overlaps it (`write.overlaps`). The last two spare any fetch whose
|
|
// lifetime overlaps this one, so concurrent over-budget fetches all keep the
|
|
// paths they return; when only such originals remain the cache stays
|
|
// over-limit until they settle, and a later, non-overlapping write finds
|
|
// them eligible again.
|
|
private enforceOriginalsLimit(write: OriginalWrite): Promise<void> {
|
|
return this.serializeEnforce(async () => {
|
|
const { entries, used } = await this.measureOriginals();
|
|
const limit = await this.originalsLimit(used);
|
|
let remaining = used;
|
|
if (remaining > limit) {
|
|
const evictable = entries
|
|
.filter(
|
|
(e) =>
|
|
e.fileID !== write.fileID &&
|
|
!write.overlaps.has(e.fileID) &&
|
|
!this.isPinned(e.fileID),
|
|
)
|
|
.sort((a, b) => a.mtimeMs - b.mtimeMs);
|
|
for (const e of evictable) {
|
|
if (remaining <= limit) break;
|
|
await rm(e.path, { force: true });
|
|
// A live photo goes whole: its video and the JSON file
|
|
// naming the two go with its image.
|
|
if (e.videoPath !== undefined) {
|
|
await rm(e.videoPath, { force: true });
|
|
await rm(
|
|
join(
|
|
this.originalsDir,
|
|
livePhotoJSONName(String(e.fileID)),
|
|
),
|
|
{ force: true },
|
|
);
|
|
}
|
|
this.originals.delete(e.fileID);
|
|
remaining -= e.size;
|
|
}
|
|
}
|
|
this.originalsUsedBytes = remaining;
|
|
this.originalsLimitBytes = limit;
|
|
});
|
|
}
|
|
|
|
// Run limit work one at a time; failures are swallowed so a transient
|
|
// statfs or unlink error never rejects the read or write that triggered it.
|
|
private serializeEnforce(work: () => Promise<void>): Promise<void> {
|
|
const next = this.enforcing.then(work).catch(() => undefined);
|
|
this.enforcing = next;
|
|
return next;
|
|
}
|
|
|
|
private async ensureDir(dir: string): Promise<void> {
|
|
// chmod after mkdir so the mode is tightened even when the directory
|
|
// already existed with a looser one; mkdir alone would not.
|
|
await mkdir(dir, { recursive: true, mode: DIR_MODE });
|
|
await chmod(dir, DIR_MODE);
|
|
}
|
|
|
|
private async scan(
|
|
dir: string,
|
|
into: Map<number, { path: string; videoPath?: string }>,
|
|
isLivePhoto: (fileID: number) => boolean,
|
|
): Promise<void> {
|
|
// Another process sharing this cache may still be writing its temp
|
|
// files, so only those whose process has exited are removed.
|
|
removeLeftoverTempFiles(dir);
|
|
let entries: string[];
|
|
try {
|
|
entries = await readdir(dir);
|
|
} catch {
|
|
return;
|
|
}
|
|
const names = new Set(entries);
|
|
for (const name of entries) {
|
|
const id = fileIDFromName(name);
|
|
const path = join(dir, name);
|
|
if (id === undefined || !existsSync(path)) continue;
|
|
// A live photo's image and video are one entry, as the JSON file
|
|
// beside them names them. A live photo's file with no such JSON
|
|
// file is not its original and is left alone: another process may
|
|
// have just stored it and not yet written the JSON file.
|
|
const livePhoto = names.has(livePhotoJSONName(String(id)))
|
|
? readLivePhotoJSON(dir, String(id))
|
|
: undefined;
|
|
if (livePhoto !== undefined) {
|
|
into.set(id, livePhoto);
|
|
} else if (!isLivePhoto(id)) {
|
|
into.set(id, { path });
|
|
}
|
|
}
|
|
}
|
|
}
|