On-disk content and thumbnail cache with per-photo fetch and prefetch (closes #46)

Add src/library/content.ts: a ContentCache keyed by fileID under
cacheDirectory (flat originals/ and thumbnails/, 0700/0600). It fetches
through the request pools (#45) and the streaming decrypt / atomic writer
(#40), so present-means-complete; it reaps orphan temp files at open and
records the record of what is cached. Photo.original/thumbnail return
{path,bytes} (one skipped event when present, in cache or downloadDirectory),
and lib.thumbnails.ensure drives the thumbnail pool with priority, dedup, and
abort. Cached paths are projected onto PhotoRecord.

Integrity: enforced by the reused streaming decrypt (every chunk
authenticated, renamed in only on TAG_FINAL) plus a non-empty check. The
metadata.hash / declared-fileSize comparison is deferred — the only in-repo
hash fixture is a placeholder, so the construction cannot be confirmed against
fixtures (the issue's own precondition), and FileBlob.size is the encrypted
object size, not the decrypted length. Raised as a question on the PR.

Judgement call: the three thumbnail priorities map onto the pool's two tiers
(visible -> on-demand; ahead and background -> background).

Model: opus-4-8

test/library/content.test.ts

@@ -0,0 +1,423 @@
+/**
+ * Tests for the on-disk content and thumbnail cache (issue #46).
+ *
+ * The cache keys stored bytes by `fileID` under `cacheDirectory`:
+ * `originals/<fileID>.<ext>` and `thumbnails/<fileID>.<ext>`. Its contract:
+ *
+ * 1. **Fetch once, then serve from disk.** The first `original`/`thumbnail`
+ *    fetches through the request pool and stores the bytes; the next finds the
+ *    file present and returns its path with a single `skipped` event and no
+ *    network. A file already sitting in the backup `downloadDirectory` counts
+ *    as present too.
+ * 2. **Present-means-complete.** Content appears only by the streaming atomic
+ *    writer's rename, so a file that exists is whole. The directory listing
+ *    taken at `open()` is the record of what is cached, and the orphan temp
+ *    files a crashed write may have left are reaped there.
+ * 3. **`thumbnails.ensure` drives the thumbnail pool with priority, dedup, and
+ *    abort.** A `fileID` asked for twice downloads once; a visible request is
+ *    served ahead of a background one; and an `AbortSignal` drops work still
+ *    queued while letting an in-flight fetch finish.
+ *
+ * The `ContentSource` is a stand-in: it writes deterministic bytes to the
+ * destination and returns the count, so the cache logic is exercised with no
+ * crypto and no network. Ordering tests gate the stand-in on explicit deferreds
+ * and assert the persisted result, never a bare call or a timer.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import {
+    mkdtempSync,
+    rmSync,
+    existsSync,
+    writeFileSync,
+    mkdirSync,
+    statSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import {
+    ContentCache,
+    type ContentSource,
+    type EnsureEvent,
+} from "../../src/library/content.js";
+import { RequestPools } from "../../src/library/pools.js";
+import type { EnteFile } from "../../src/model/types.js";
+
+const file = (id: number, title = `file-${id}.jpg`): EnteFile => ({
+    id,
+    collectionID: 1,
+    ownerID: 1,
+    key: new Uint8Array([id & 0xff]),
+    metadata: {
+        title,
+        fileType: "image",
+        creationTime: 0,
+        modificationTime: 0,
+    },
+    file: { decryptionHeader: "aGVhZGVy" },
+    thumbnail: { decryptionHeader: "dGh1bWI=" },
+    updationTime: 0,
+});
+
+// A deferred with externally callable resolve, used to gate the stand-in source
+// so ordering is controlled by the test rather than by timing.
+const deferred = (): { promise: Promise<void>; resolve: () => void } => {
+    let resolve!: () => void;
+    const promise = new Promise<void>((r) => {
+        resolve = r;
+    });
+    return { promise, resolve };
+};
+
+// A ContentSource that writes `${kind}:${fileID}` bytes to the destination and
+// records every call. `gate` optionally blocks a call until released, and
+// `completed` records the order in which fetches finished — the observable used
+// by the priority and abort tests instead of a timer.
+class StubSource implements ContentSource {
+    originalCalls: number[] = [];
+    thumbnailCalls: number[] = [];
+    completed: number[] = [];
+    emptyFor = new Set<number>();
+    gates = new Map<number, Promise<void>>();
+
+    private async run(
+        kind: "original" | "thumbnail",
+        file: EnteFile,
+        destination: string,
+    ): Promise<{ bytesWritten: number }> {
+        const gate = this.gates.get(file.id);
+        if (gate) await gate;
+        const bytes = this.emptyFor.has(file.id)
+            ? new Uint8Array(0)
+            : new TextEncoder().encode(`${kind}:${file.id}`);
+        writeFileSync(destination, bytes);
+        this.completed.push(file.id);
+        return { bytesWritten: bytes.length };
+    }
+
+    async original(args: {
+        file: EnteFile;
+        destination: string;
+    }): Promise<{ bytesWritten: number }> {
+        this.originalCalls.push(args.file.id);
+        return this.run("original", args.file, args.destination);
+    }
+
+    async thumbnail(args: {
+        file: EnteFile;
+        destination: string;
+    }): Promise<{ bytesWritten: number }> {
+        this.thumbnailCalls.push(args.file.id);
+        return this.run("thumbnail", args.file, args.destination);
+    }
+}
+
+let root: string;
+let cacheDir: string;
+
+beforeEach(() => {
+    root = mkdtempSync(join(tmpdir(), "quak-content-"));
+    cacheDir = join(root, "cache");
+});
+
+afterEach(() => {
+    if (root && existsSync(root))
+        rmSync(root, { recursive: true, force: true });
+});
+
+const buildCache = (
+    args: {
+        source?: ContentSource;
+        files?: EnteFile[];
+        pools?: RequestPools;
+        downloadDirectory?: string;
+    } = {},
+): { cache: ContentCache; source: StubSource } => {
+    const source = (args.source as StubSource) ?? new StubSource();
+    const byID = new Map<number, EnteFile>();
+    for (const f of args.files ?? [file(1), file(2), file(3)])
+        byID.set(f.id, f);
+    const cache = new ContentCache({
+        pools: args.pools ?? new RequestPools(),
+        source,
+        cacheDirectory: cacheDir,
+        downloadDirectory: args.downloadDirectory,
+        getFile: (id) => byID.get(id),
+    });
+    return { cache, source };
+};
+
+describe("ContentCache.open", () => {
+    it("creates the cache directories with 0700 permissions", async () => {
+        const { cache } = buildCache();
+        await cache.open();
+
+        const originals = join(cacheDir, "originals");
+        const thumbnails = join(cacheDir, "thumbnails");
+        expect(existsSync(originals)).toBe(true);
+        expect(existsSync(thumbnails)).toBe(true);
+        expect(statSync(originals).mode & 0o777).toBe(0o700);
+        expect(statSync(thumbnails).mode & 0o777).toBe(0o700);
+    });
+
+    it("reaps orphan temp files but keeps complete content", async () => {
+        const originals = join(cacheDir, "originals");
+        const thumbnails = join(cacheDir, "thumbnails");
+        mkdirSync(originals, { recursive: true });
+        mkdirSync(thumbnails, { recursive: true });
+        const orphan = join(originals, ".quak-abc123.tmp");
+        const complete = join(originals, "1.jpg");
+        const thumb = join(thumbnails, "2.jpg");
+        writeFileSync(orphan, "half-written");
+        writeFileSync(complete, "whole");
+        writeFileSync(thumb, "whole-thumb");
+
+        const { cache } = buildCache();
+        await cache.open();
+
+        expect(existsSync(orphan)).toBe(false);
+        expect(existsSync(complete)).toBe(true);
+        expect(existsSync(thumb)).toBe(true);
+    });
+
+    it("records already-cached files so their paths appear in pathsFor", async () => {
+        const originals = join(cacheDir, "originals");
+        const thumbnails = join(cacheDir, "thumbnails");
+        mkdirSync(originals, { recursive: true });
+        mkdirSync(thumbnails, { recursive: true });
+        writeFileSync(join(originals, "1.jpg"), "orig");
+        writeFileSync(join(thumbnails, "1.jpg"), "thumb");
+
+        const { cache } = buildCache();
+        await cache.open();
+
+        expect(cache.pathsFor(1)).toEqual({
+            originalPath: join(originals, "1.jpg"),
+            thumbnailPath: join(thumbnails, "1.jpg"),
+        });
+        expect(cache.pathsFor(2)).toEqual({});
+    });
+});
+
+describe("ContentCache.original / thumbnail", () => {
+    it("fetches once, then serves the cached file with a single skipped event", async () => {
+        const { cache, source } = buildCache();
+        await cache.open();
+
+        const events: string[] = [];
+        const first = await cache.original(1, {
+            onProgress: (e) => events.push(e.status),
+        });
+        expect(source.originalCalls).toEqual([1]);
+        expect(first.path).toBe(join(cacheDir, "originals", "1.jpg"));
+        expect(first.bytes).toBe("original:1".length);
+        expect(existsSync(first.path)).toBe(true);
+        expect(statSync(first.path).mode & 0o777).toBe(0o600);
+        expect(cache.pathsFor(1).originalPath).toBe(first.path);
+
+        const skips: string[] = [];
+        const second = await cache.original(1, {
+            onProgress: (e) => skips.push(e.status),
+        });
+        // No second download, and exactly one skipped event.
+        expect(source.originalCalls).toEqual([1]);
+        expect(second.path).toBe(first.path);
+        expect(skips).toEqual(["skipped"]);
+    });
+
+    it("serves a file already present in the download directory without fetching", async () => {
+        const downloadDirectory = join(root, "backup");
+        mkdirSync(join(downloadDirectory, "originals"), { recursive: true });
+        const backupPath = join(downloadDirectory, "originals", "1.jpg");
+        writeFileSync(backupPath, "from-backup");
+
+        const { cache, source } = buildCache({ downloadDirectory });
+        await cache.open();
+
+        const events: EnsureEvent["status"][] = [];
+        const result = await cache.original(1, {
+            onProgress: (e) => events.push(e.status),
+        });
+
+        expect(source.originalCalls).toEqual([]);
+        expect(result.path).toBe(backupPath);
+        expect(result.bytes).toBe("from-backup".length);
+        expect(events).toEqual(["skipped"]);
+    });
+
+    it("fetches and caches a thumbnail", async () => {
+        const { cache, source } = buildCache();
+        await cache.open();
+
+        const result = await cache.thumbnail(2);
+        expect(source.thumbnailCalls).toEqual([2]);
+        expect(result.path).toBe(join(cacheDir, "thumbnails", "2.jpg"));
+        expect(existsSync(result.path)).toBe(true);
+        expect(cache.pathsFor(2).thumbnailPath).toBe(result.path);
+    });
+
+    it("shares one download between concurrent callers for the same file", async () => {
+        const { cache, source } = buildCache();
+        await cache.open();
+        const gate = deferred();
+        source.gates.set(1, gate.promise);
+
+        const a = cache.original(1);
+        const b = cache.original(1);
+        gate.resolve();
+        const [ra, rb] = await Promise.all([a, b]);
+
+        expect(source.originalCalls).toEqual([1]);
+        expect(ra.path).toBe(rb.path);
+    });
+
+    it("does not record a path when the fetched file is empty", async () => {
+        const { cache, source } = buildCache();
+        source.emptyFor.add(1);
+        await cache.open();
+
+        await expect(cache.original(1)).rejects.toThrow(/empty/i);
+        expect(cache.pathsFor(1).originalPath).toBeUndefined();
+    });
+
+    it("rejects an unknown file", async () => {
+        const { cache } = buildCache({ files: [] });
+        await cache.open();
+        await expect(cache.original(999)).rejects.toThrow(/unknown file/i);
+    });
+});
+
+describe("ContentCache.ensureThumbnails", () => {
+    it("downloads once for a file listed twice and reports every id", async () => {
+        const { cache, source } = buildCache();
+        await cache.open();
+
+        const results = await cache.ensureThumbnails({
+            fileIDs: [1, 1, 2],
+            priority: "visible",
+        });
+
+        expect(source.thumbnailCalls.sort()).toEqual([1, 2]);
+        expect(results).toEqual([
+            { fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") },
+            { fileID: 2, path: join(cacheDir, "thumbnails", "2.jpg") },
+        ]);
+    });
+
+    it("skips present files and reports a skipped event", async () => {
+        const thumbnails = join(cacheDir, "thumbnails");
+        mkdirSync(thumbnails, { recursive: true });
+        writeFileSync(join(thumbnails, "1.jpg"), "present");
+
+        const { cache, source } = buildCache();
+        await cache.open();
+
+        const events: EnsureEvent[] = [];
+        const results = await cache.ensureThumbnails({
+            fileIDs: [1, 2],
+            priority: "ahead",
+            onProgress: (e) => events.push(e),
+        });
+
+        expect(source.thumbnailCalls).toEqual([2]);
+        expect(results).toEqual([
+            { fileID: 1, path: join(thumbnails, "1.jpg") },
+            { fileID: 2, path: join(thumbnails, "2.jpg") },
+        ]);
+        expect(events).toContainEqual({
+            fileID: 1,
+            status: "skipped",
+            path: join(thumbnails, "1.jpg"),
+        });
+    });
+
+    it("serves a visible request ahead of an already-queued background one", async () => {
+        // One thumbnail slot, so exactly one fetch runs at a time and the rest
+        // wait in the pool. A background fetch takes the slot; a background and
+        // a visible fetch queue behind it. When the slot frees, the pool must
+        // pick the visible (on-demand) request ahead of the background one that
+        // was submitted first. The completion order is the observable.
+        const pools = new RequestPools({ thumbnailConcurrency: 1 });
+        const { cache, source } = buildCache({ pools });
+        await cache.open();
+
+        const gateA = deferred();
+        const gateB = deferred();
+        const gateC = deferred();
+        source.gates.set(1, gateA.promise);
+        source.gates.set(2, gateB.promise);
+        source.gates.set(3, gateC.promise);
+
+        const bgFirst = cache.ensureThumbnails({
+            fileIDs: [1],
+            priority: "background",
+        });
+        // Let fetch 1 take the only slot before the others queue.
+        await Promise.resolve();
+        const bgSecond = cache.ensureThumbnails({
+            fileIDs: [2],
+            priority: "background",
+        });
+        const visible = cache.ensureThumbnails({
+            fileIDs: [3],
+            priority: "visible",
+        });
+
+        gateA.resolve();
+        gateC.resolve();
+        gateB.resolve();
+        await Promise.all([bgFirst, bgSecond, visible]);
+
+        // 1 ran first (it held the slot). Of the two that were queued, the
+        // visible id 3 was served before the background id 2.
+        expect(source.completed).toEqual([1, 3, 2]);
+    });
+
+    it("drops queued work on abort but keeps an in-flight fetch", async () => {
+        const pools = new RequestPools({ thumbnailConcurrency: 1 });
+        const { cache, source } = buildCache({ pools });
+        await cache.open();
+
+        const gate = deferred();
+        source.gates.set(1, gate.promise);
+        const controller = new AbortController();
+
+        const pending = cache.ensureThumbnails({
+            fileIDs: [1, 2],
+            priority: "ahead",
+            signal: controller.signal,
+        });
+        // Fetch 1 is in flight (holds the slot); 2 is queued.
+        await Promise.resolve();
+        controller.abort();
+        gate.resolve();
+
+        const results = await pending;
+
+        // The in-flight fetch finished and is kept; the queued one was dropped
+        // before it ran.
+        expect(source.thumbnailCalls).toEqual([1]);
+        expect(results).toEqual([
+            { fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") },
+            { fileID: 2, error: "aborted" },
+        ]);
+    });
+
+    it("captures a per-file failure without failing the batch", async () => {
+        const { cache } = buildCache({ files: [file(1)] });
+        await cache.open();
+
+        const results = await cache.ensureThumbnails({
+            fileIDs: [1, 2],
+            priority: "background",
+        });
+
+        expect(results[0]).toEqual({
+            fileID: 1,
+            path: join(cacheDir, "thumbnails", "1.jpg"),
+        });
+        expect(results[1]?.fileID).toBe(2);
+        expect(results[1]?.error).toMatch(/unknown file/i);
+    });
+});