sneak
8e06cd5723
check / check (push) Waiting to run
`quak files COLLECTION` used to wait for a full refresh, which on an empty cache fetched the files of every collection before it listed one. It now reads the collection list, then calls the new `Library.refreshCollection()`, which brings that one collection up to date in the store as a refresh would, and saves it. It sends no request for the files when the stored copy is current. It leaves the refresh cursor alone, so the next refresh still fetches every other collection that changed. `refreshOnce` and `refreshCollection` share the commit step. `quak files` with no argument still waits for a refresh. Model: opus-5-5
428 lines
15 KiB
TypeScript
428 lines
15 KiB
TypeScript
/**
|
|
* Tests for fresh reads (issue #75, an owner amendment to design #36).
|
|
*
|
|
* The default read namespaces answer from RAM and never touch the network; a
|
|
* background loop keeps the local copy current. `fresh()` adds an awaited path:
|
|
* it forces a refresh, waits for it to complete and persist, and only then
|
|
* hands back the `albums`/`photos`/`timeline` namespaces, guaranteeing the
|
|
* local copy reflects a completed server round-trip. The contracts here:
|
|
*
|
|
* 1. A fresh read observes a server change that a same-instant default read
|
|
* would miss (the default path has not refreshed yet).
|
|
* 2. Concurrent fresh reads coalesce onto one in-flight refresh — three
|
|
* concurrent `fresh()` calls make exactly one collections round-trip, not
|
|
* three.
|
|
* 3. A refresh that fails rejects the fresh read (currency was unavailable),
|
|
* while the default reads stay silent and keep serving the last good copy.
|
|
* 4. `freshCollections()` reads the collection list alone, with one request
|
|
* and no collection's files, and stores nothing, so the next refresh still
|
|
* fetches the files of every collection (issue #210).
|
|
* 5. `refreshCollection()` brings one collection up to date in the store and
|
|
* leaves the stored cursor where it was, so the next refresh still fetches
|
|
* the files of every other collection that changed (issue #217).
|
|
*
|
|
* The client is the same metadata-only mock the background-refresh tests use:
|
|
* no crypto, no network, scripted pages, and a record of each call. A long
|
|
* refresh interval keeps the background timer out of the way so each test's
|
|
* refreshes are exactly the ones it triggers.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
import { Library } from "../../src/library/index.js";
|
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
|
|
|
const USER_ID = 42;
|
|
|
|
// Long enough that no background tick fires during a test; each test's
|
|
// refreshes are only the ones its own `fresh()` calls force.
|
|
const SLOW_INTERVAL = 3600;
|
|
|
|
const collection = (id: number, updationTime: number): Collection => ({
|
|
id,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
name: `album-${id}`,
|
|
type: "album",
|
|
updationTime,
|
|
isShared: false,
|
|
});
|
|
|
|
const file = (
|
|
id: number,
|
|
collectionID: number,
|
|
updationTime: number,
|
|
): EnteFile => ({
|
|
id,
|
|
collectionID,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
metadata: {
|
|
title: `file-${id}.jpg`,
|
|
fileType: "image",
|
|
creationTime: updationTime,
|
|
modificationTime: updationTime,
|
|
},
|
|
file: { decryptionHeader: "aGVhZGVy" },
|
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
|
updationTime,
|
|
});
|
|
|
|
class MockClient {
|
|
userID = USER_ID;
|
|
failCollections = false;
|
|
collectionsQueue: CollectionsPage[] = [];
|
|
filesByCollection = new Map<number, FilesPage[]>();
|
|
|
|
collectionsSinceTimes: number[] = [];
|
|
filesCalls: { collectionID: number; sinceTime: number }[] = [];
|
|
|
|
whoami(): { email: string; userID: number } {
|
|
return { email: "user@example.com", userID: this.userID };
|
|
}
|
|
|
|
async collectionsSince(args: {
|
|
sinceTime: number;
|
|
}): Promise<CollectionsPage> {
|
|
this.collectionsSinceTimes.push(args.sinceTime);
|
|
if (this.failCollections) throw new Error("network down");
|
|
return (
|
|
this.collectionsQueue.shift() ?? {
|
|
collections: [],
|
|
deleted: [],
|
|
cursor: args.sinceTime,
|
|
}
|
|
);
|
|
}
|
|
|
|
async filesSince(args: {
|
|
collectionID: number;
|
|
collectionKey: Uint8Array;
|
|
sinceTime: number;
|
|
}): Promise<FilesPage> {
|
|
this.filesCalls.push({
|
|
collectionID: args.collectionID,
|
|
sinceTime: args.sinceTime,
|
|
});
|
|
const queue = this.filesByCollection.get(args.collectionID);
|
|
return (
|
|
queue?.shift() ?? {
|
|
files: [],
|
|
deleted: [],
|
|
cursor: args.sinceTime,
|
|
}
|
|
);
|
|
}
|
|
|
|
filesFor(collectionID: number, ...pages: FilesPage[]): void {
|
|
this.filesByCollection.set(collectionID, pages);
|
|
}
|
|
}
|
|
|
|
// A client seeded with one collection and one file, opened with an empty cache
|
|
// so the initial refresh is awaited and post-`open()` state is deterministic.
|
|
const openSeeded = async (
|
|
cacheDirectory: string,
|
|
): Promise<{ client: MockClient; lib: Library }> => {
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: SLOW_INTERVAL,
|
|
});
|
|
return { client, lib };
|
|
};
|
|
|
|
describe("Library.fresh", () => {
|
|
let dir: string;
|
|
let cacheDirectory: string;
|
|
|
|
beforeEach(() => {
|
|
dir = mkdtempSync(join(tmpdir(), "quak-fresh-"));
|
|
cacheDirectory = join(dir, "cache");
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("observes a server change a same-instant default read would miss", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
try {
|
|
// A new file appears on the server after open, advancing its
|
|
// collection so the next refresh re-enumerates it.
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 200)],
|
|
deleted: [],
|
|
cursor: 200,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 190)],
|
|
deleted: [],
|
|
cursor: 190,
|
|
});
|
|
|
|
// A default read at this instant has not refreshed: it misses 1002.
|
|
expect(lib.photos.byID({ fileID: 1002 })).toBeUndefined();
|
|
|
|
// A fresh read forces the round-trip and sees it.
|
|
const reads = await lib.fresh();
|
|
expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
|
// And the change is now live for the default namespaces too.
|
|
expect(lib.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("coalesces concurrent fresh reads onto one in-flight refresh", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
try {
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 200)],
|
|
deleted: [],
|
|
cursor: 200,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 190)],
|
|
deleted: [],
|
|
cursor: 190,
|
|
});
|
|
|
|
const collectionsBefore = client.collectionsSinceTimes.length;
|
|
const filesBefore = client.filesCalls.length;
|
|
|
|
// Gate the next collections fetch so all three fresh reads are in
|
|
// flight together before any of them completes.
|
|
let release: () => void = () => {};
|
|
const gate = new Promise<void>((r) => {
|
|
release = r;
|
|
});
|
|
const inner = client.collectionsSince.bind(client);
|
|
client.collectionsSince = async (args: { sinceTime: number }) => {
|
|
await gate;
|
|
return inner(args);
|
|
};
|
|
|
|
const all = Promise.all([lib.fresh(), lib.fresh(), lib.fresh()]);
|
|
release();
|
|
const [a, b, c] = await all;
|
|
|
|
// Exactly one collections round-trip and one file round-trip served
|
|
// all three fresh reads.
|
|
expect(client.collectionsSinceTimes.length).toBe(
|
|
collectionsBefore + 1,
|
|
);
|
|
expect(client.filesCalls.length).toBe(filesBefore + 1);
|
|
|
|
// All three observed the change.
|
|
for (const reads of [a, b, c]) {
|
|
expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
|
}
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("rejects the fresh read when the refresh fails, leaving defaults intact", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
try {
|
|
client.failCollections = true;
|
|
|
|
// Two concurrent fresh reads both reject, and share one failed
|
|
// round-trip rather than each making its own.
|
|
const collectionsBefore = client.collectionsSinceTimes.length;
|
|
const first = lib.fresh();
|
|
const second = lib.fresh();
|
|
await expect(first).rejects.toThrow(/network down/);
|
|
await expect(second).rejects.toThrow(/network down/);
|
|
expect(client.collectionsSinceTimes.length).toBe(
|
|
collectionsBefore + 1,
|
|
);
|
|
|
|
// The default reads never rejected: they still serve the last good
|
|
// copy, and the failure surfaced through status().
|
|
expect(lib.photos.byID({ fileID: 1001 })?.fileID).toBe(1001);
|
|
expect(lib.status().lastError).toMatch(/network down/);
|
|
|
|
// Recovery: once the server answers, a fresh read resolves current.
|
|
client.failCollections = false;
|
|
client.collectionsQueue.push({
|
|
collections: [collection(2, 300)],
|
|
deleted: [],
|
|
cursor: 300,
|
|
});
|
|
const reads = await lib.fresh();
|
|
expect(reads.albums.byID({ collectionID: 2 })?.collectionID).toBe(
|
|
2,
|
|
);
|
|
expect(lib.status().lastError).toBeUndefined();
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("Library.freshCollections", () => {
|
|
let dir: string;
|
|
let cacheDirectory: string;
|
|
|
|
beforeEach(() => {
|
|
dir = mkdtempSync(join(tmpdir(), "quak-fresh-collections-"));
|
|
cacheDirectory = join(dir, "cache");
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
// A refresh fetches a collection's files only when the collection is new
|
|
// or its updationTime has advanced past the stored copy's. Had the
|
|
// collection list been stored, the next refresh would take every
|
|
// collection for one it already holds the files of, and fetch none.
|
|
it("asks for the collection list alone and stores nothing", async () => {
|
|
const client = new MockClient();
|
|
const page = {
|
|
collections: [collection(2, 100), collection(1, 110)],
|
|
deleted: [],
|
|
cursor: 110,
|
|
};
|
|
client.collectionsQueue.push(page, page);
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
client.filesFor(2, {
|
|
files: [file(2001, 2, 95)],
|
|
deleted: [],
|
|
cursor: 95,
|
|
});
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshOnOpen: false,
|
|
refreshIntervalSeconds: SLOW_INTERVAL,
|
|
});
|
|
try {
|
|
// One request, the whole list in the server's order, and no
|
|
// collection's files.
|
|
const collections = await lib.freshCollections();
|
|
expect(collections.map((c) => c.id)).toEqual([2, 1]);
|
|
expect(collections.map((c) => c.name)).toEqual([
|
|
"album-2",
|
|
"album-1",
|
|
]);
|
|
expect(client.collectionsSinceTimes).toEqual([0]);
|
|
expect(client.filesCalls).toEqual([]);
|
|
expect(lib.listCollections()).toEqual([]);
|
|
|
|
// The next refresh still asks from the beginning and fetches the
|
|
// files of both collections.
|
|
const reads = await lib.fresh();
|
|
expect(client.collectionsSinceTimes).toEqual([0, 0]);
|
|
expect(reads.photos.byID({ fileID: 1001 })?.fileID).toBe(1001);
|
|
expect(reads.photos.byID({ fileID: 2001 })?.fileID).toBe(2001);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("Library.refreshCollection", () => {
|
|
let dir: string;
|
|
let cacheDirectory: string;
|
|
|
|
beforeEach(() => {
|
|
dir = mkdtempSync(join(tmpdir(), "quak-refresh-collection-"));
|
|
cacheDirectory = join(dir, "cache");
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
// Collections 1 and 2 are stored at updationTime 100, with a stored cursor
|
|
// of 100, and then both change on the server.
|
|
it("fetches the files of one changed collection and leaves the others to the next refresh", async () => {
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100), collection(2, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
client.filesFor(2, {
|
|
files: [file(2001, 2, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: SLOW_INTERVAL,
|
|
});
|
|
try {
|
|
client.filesCalls.length = 0;
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 110)],
|
|
deleted: [],
|
|
cursor: 110,
|
|
});
|
|
client.filesFor(2, {
|
|
files: [file(2002, 2, 120)],
|
|
deleted: [],
|
|
cursor: 120,
|
|
});
|
|
|
|
// Collection 1's files alone, since the stored copy.
|
|
await lib.refreshCollection(collection(1, 110));
|
|
expect(client.filesCalls).toEqual([
|
|
{ collectionID: 1, sinceTime: 100 },
|
|
]);
|
|
expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001, 1002]);
|
|
expect(lib.listFiles(2).map((f) => f.id)).toEqual([2001]);
|
|
|
|
// The stored copy is now current, so no request.
|
|
await lib.refreshCollection(collection(1, 110));
|
|
expect(client.filesCalls).toHaveLength(1);
|
|
|
|
// The next refresh asks from the stored cursor, 100, so it still
|
|
// sees collection 2 changed and fetches its files.
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 110), collection(2, 120)],
|
|
deleted: [],
|
|
cursor: 120,
|
|
});
|
|
await lib.fresh();
|
|
expect(client.collectionsSinceTimes).toEqual([0, 100]);
|
|
expect(client.filesCalls).toEqual([
|
|
{ collectionID: 1, sinceTime: 100 },
|
|
{ collectionID: 2, sinceTime: 100 },
|
|
]);
|
|
expect(lib.listFiles(2).map((f) => f.id)).toEqual([2001, 2002]);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
});
|