sneak
8c91fb8aaa
check / check (push) Waiting to run
`quak files COLLECTION` used to wait for a full refresh, which on an empty cache fetched the files of every collection first. It now reads the collection list, then calls the new `Library.refreshCollection()`. That brings one collection up to date in the store as a refresh would. Once the stored cursor is past 0, it only updates a collection the store holds. For a new one, `quak files COLLECTION` waits for a refresh. A refresh from a cursor of 0 replaces all the store holds, so collections keep the server's order. `open()` awaits the first refresh while the cursor is 0. The store lists what did not decrypt in a fixed order. Model: opus-5-5
612 lines
22 KiB
TypeScript
612 lines
22 KiB
TypeScript
/**
|
|
* Tests for fresh reads (issue #75, an owner amendment to design #36).
|
|
*
|
|
* The default read namespaces answer from RAM and never touch the network; a
|
|
* background loop keeps the local copy current. `fresh()` adds an awaited path:
|
|
* it forces a refresh, waits for it to complete and persist, and only then
|
|
* hands back the `albums`/`photos`/`timeline` namespaces, guaranteeing the
|
|
* local copy reflects a completed server round-trip. The contracts here:
|
|
*
|
|
* 1. A fresh read observes a server change that a same-instant default read
|
|
* would miss (the default path has not refreshed yet).
|
|
* 2. Concurrent fresh reads coalesce onto one in-flight refresh — three
|
|
* concurrent `fresh()` calls make exactly one collections round-trip, not
|
|
* three.
|
|
* 3. A refresh that fails rejects the fresh read (currency was unavailable),
|
|
* while the default reads stay silent and keep serving the last good copy.
|
|
* 4. `freshCollections()` reads the collection list alone, with one request
|
|
* and no collection's files, and stores nothing, so the next refresh still
|
|
* fetches the files of every collection (issue #210).
|
|
* 5. `refreshCollection()` brings one collection up to date in the store
|
|
* (issue #217). It leaves the stored cursor where it was, so the next
|
|
* refresh still fetches the files of every other collection that changed.
|
|
* It never runs at the same time as a refresh. After a refresh, it stores
|
|
* no collection the store does not hold: not a new one, and not one a
|
|
* refresh deleted. `close()` waits for it. A library opened on a cache that
|
|
* only it wrote awaits its first refresh. That refresh replaces what it
|
|
* stored, in the server's order.
|
|
*
|
|
* The client is the same metadata-only mock the background-refresh tests use:
|
|
* no crypto, no network, scripted pages, and a record of each call. A long
|
|
* refresh interval keeps the background timer out of the way so each test's
|
|
* refreshes are exactly the ones it triggers.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
import { Library } from "../../src/library/index.js";
|
|
import { MetadataStore } from "../../src/library/store.js";
|
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
|
|
|
const USER_ID = 42;
|
|
|
|
// Long enough that no background tick fires during a test; each test's
|
|
// refreshes are only the ones its own `fresh()` calls force.
|
|
const SLOW_INTERVAL = 3600;
|
|
|
|
const collection = (id: number, updationTime: number): Collection => ({
|
|
id,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
name: `album-${id}`,
|
|
type: "album",
|
|
updationTime,
|
|
isShared: false,
|
|
});
|
|
|
|
const file = (
|
|
id: number,
|
|
collectionID: number,
|
|
updationTime: number,
|
|
): EnteFile => ({
|
|
id,
|
|
collectionID,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
metadata: {
|
|
title: `file-${id}.jpg`,
|
|
fileType: "image",
|
|
creationTime: updationTime,
|
|
modificationTime: updationTime,
|
|
},
|
|
file: { decryptionHeader: "aGVhZGVy" },
|
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
|
updationTime,
|
|
});
|
|
|
|
class MockClient {
|
|
userID = USER_ID;
|
|
failCollections = false;
|
|
collectionsQueue: CollectionsPage[] = [];
|
|
filesByCollection = new Map<number, FilesPage[]>();
|
|
|
|
collectionsSinceTimes: number[] = [];
|
|
filesCalls: { collectionID: number; sinceTime: number }[] = [];
|
|
|
|
whoami(): { email: string; userID: number } {
|
|
return { email: "user@example.com", userID: this.userID };
|
|
}
|
|
|
|
async collectionsSince(args: {
|
|
sinceTime: number;
|
|
}): Promise<CollectionsPage> {
|
|
this.collectionsSinceTimes.push(args.sinceTime);
|
|
if (this.failCollections) throw new Error("network down");
|
|
return (
|
|
this.collectionsQueue.shift() ?? {
|
|
collections: [],
|
|
deleted: [],
|
|
cursor: args.sinceTime,
|
|
}
|
|
);
|
|
}
|
|
|
|
async filesSince(args: {
|
|
collectionID: number;
|
|
collectionKey: Uint8Array;
|
|
sinceTime: number;
|
|
}): Promise<FilesPage> {
|
|
this.filesCalls.push({
|
|
collectionID: args.collectionID,
|
|
sinceTime: args.sinceTime,
|
|
});
|
|
const queue = this.filesByCollection.get(args.collectionID);
|
|
return (
|
|
queue?.shift() ?? {
|
|
files: [],
|
|
deleted: [],
|
|
cursor: args.sinceTime,
|
|
}
|
|
);
|
|
}
|
|
|
|
filesFor(collectionID: number, ...pages: FilesPage[]): void {
|
|
this.filesByCollection.set(collectionID, pages);
|
|
}
|
|
}
|
|
|
|
// A client seeded with one collection and one file, opened with an empty cache
|
|
// so the initial refresh is awaited and post-`open()` state is deterministic.
|
|
const openSeeded = async (
|
|
cacheDirectory: string,
|
|
): Promise<{ client: MockClient; lib: Library }> => {
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: SLOW_INTERVAL,
|
|
});
|
|
return { client, lib };
|
|
};
|
|
|
|
// Hold the client's next files request until `release()` is called. `started`
|
|
// resolves once that request has been sent.
|
|
const holdNextFilesRequest = (
|
|
client: MockClient,
|
|
): { started: Promise<void>; release: () => void } => {
|
|
let release!: () => void;
|
|
const held = new Promise<void>((r) => (release = r));
|
|
let sent!: () => void;
|
|
const started = new Promise<void>((r) => (sent = r));
|
|
const realFilesSince = client.filesSince.bind(client);
|
|
client.filesSince = async (args) => {
|
|
client.filesSince = realFilesSince;
|
|
const page = await realFilesSince(args);
|
|
sent();
|
|
await held;
|
|
return page;
|
|
};
|
|
return { started, release };
|
|
};
|
|
|
|
describe("Library.fresh", () => {
|
|
let dir: string;
|
|
let cacheDirectory: string;
|
|
|
|
beforeEach(() => {
|
|
dir = mkdtempSync(join(tmpdir(), "quak-fresh-"));
|
|
cacheDirectory = join(dir, "cache");
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("observes a server change a same-instant default read would miss", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
try {
|
|
// A new file appears on the server after open, advancing its
|
|
// collection so the next refresh re-enumerates it.
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 200)],
|
|
deleted: [],
|
|
cursor: 200,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 190)],
|
|
deleted: [],
|
|
cursor: 190,
|
|
});
|
|
|
|
// A default read at this instant has not refreshed: it misses 1002.
|
|
expect(lib.photos.byID({ fileID: 1002 })).toBeUndefined();
|
|
|
|
// A fresh read forces the round-trip and sees it.
|
|
const reads = await lib.fresh();
|
|
expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
|
// And the change is now live for the default namespaces too.
|
|
expect(lib.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("coalesces concurrent fresh reads onto one in-flight refresh", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
try {
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 200)],
|
|
deleted: [],
|
|
cursor: 200,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 190)],
|
|
deleted: [],
|
|
cursor: 190,
|
|
});
|
|
|
|
const collectionsBefore = client.collectionsSinceTimes.length;
|
|
const filesBefore = client.filesCalls.length;
|
|
|
|
// Gate the next collections fetch so all three fresh reads are in
|
|
// flight together before any of them completes.
|
|
let release: () => void = () => {};
|
|
const gate = new Promise<void>((r) => {
|
|
release = r;
|
|
});
|
|
const inner = client.collectionsSince.bind(client);
|
|
client.collectionsSince = async (args: { sinceTime: number }) => {
|
|
await gate;
|
|
return inner(args);
|
|
};
|
|
|
|
const all = Promise.all([lib.fresh(), lib.fresh(), lib.fresh()]);
|
|
release();
|
|
const [a, b, c] = await all;
|
|
|
|
// Exactly one collections round-trip and one file round-trip served
|
|
// all three fresh reads.
|
|
expect(client.collectionsSinceTimes.length).toBe(
|
|
collectionsBefore + 1,
|
|
);
|
|
expect(client.filesCalls.length).toBe(filesBefore + 1);
|
|
|
|
// All three observed the change.
|
|
for (const reads of [a, b, c]) {
|
|
expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
|
}
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("rejects the fresh read when the refresh fails, leaving defaults intact", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
try {
|
|
client.failCollections = true;
|
|
|
|
// Two concurrent fresh reads both reject, and share one failed
|
|
// round-trip rather than each making its own.
|
|
const collectionsBefore = client.collectionsSinceTimes.length;
|
|
const first = lib.fresh();
|
|
const second = lib.fresh();
|
|
await expect(first).rejects.toThrow(/network down/);
|
|
await expect(second).rejects.toThrow(/network down/);
|
|
expect(client.collectionsSinceTimes.length).toBe(
|
|
collectionsBefore + 1,
|
|
);
|
|
|
|
// The default reads never rejected: they still serve the last good
|
|
// copy, and the failure surfaced through status().
|
|
expect(lib.photos.byID({ fileID: 1001 })?.fileID).toBe(1001);
|
|
expect(lib.status().lastError).toMatch(/network down/);
|
|
|
|
// Recovery: once the server answers, a fresh read resolves current.
|
|
client.failCollections = false;
|
|
client.collectionsQueue.push({
|
|
collections: [collection(2, 300)],
|
|
deleted: [],
|
|
cursor: 300,
|
|
});
|
|
const reads = await lib.fresh();
|
|
expect(reads.albums.byID({ collectionID: 2 })?.collectionID).toBe(
|
|
2,
|
|
);
|
|
expect(lib.status().lastError).toBeUndefined();
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("Library.freshCollections", () => {
|
|
let dir: string;
|
|
let cacheDirectory: string;
|
|
|
|
beforeEach(() => {
|
|
dir = mkdtempSync(join(tmpdir(), "quak-fresh-collections-"));
|
|
cacheDirectory = join(dir, "cache");
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
// A refresh fetches a collection's files only when the collection is new
|
|
// or its updationTime has advanced past the stored copy's. Had the
|
|
// collection list been stored, the next refresh would take every
|
|
// collection for one it already holds the files of, and fetch none.
|
|
it("asks for the collection list alone and stores nothing", async () => {
|
|
const client = new MockClient();
|
|
const page = {
|
|
collections: [collection(2, 100), collection(1, 110)],
|
|
deleted: [],
|
|
cursor: 110,
|
|
};
|
|
client.collectionsQueue.push(page, page);
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
client.filesFor(2, {
|
|
files: [file(2001, 2, 95)],
|
|
deleted: [],
|
|
cursor: 95,
|
|
});
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshOnOpen: false,
|
|
refreshIntervalSeconds: SLOW_INTERVAL,
|
|
});
|
|
try {
|
|
// One request, the whole list in the server's order, and no
|
|
// collection's files.
|
|
const collections = await lib.freshCollections();
|
|
expect(collections.map((c) => c.id)).toEqual([2, 1]);
|
|
expect(collections.map((c) => c.name)).toEqual([
|
|
"album-2",
|
|
"album-1",
|
|
]);
|
|
expect(client.collectionsSinceTimes).toEqual([0]);
|
|
expect(client.filesCalls).toEqual([]);
|
|
expect(lib.listCollections()).toEqual([]);
|
|
|
|
// The next refresh still asks from the beginning and fetches the
|
|
// files of both collections.
|
|
const reads = await lib.fresh();
|
|
expect(client.collectionsSinceTimes).toEqual([0, 0]);
|
|
expect(reads.photos.byID({ fileID: 1001 })?.fileID).toBe(1001);
|
|
expect(reads.photos.byID({ fileID: 2001 })?.fileID).toBe(2001);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("Library.refreshCollection", () => {
|
|
let dir: string;
|
|
let cacheDirectory: string;
|
|
|
|
beforeEach(() => {
|
|
dir = mkdtempSync(join(tmpdir(), "quak-refresh-collection-"));
|
|
cacheDirectory = join(dir, "cache");
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
// Collections 1 and 2 are stored at updationTime 100, with a stored cursor
|
|
// of 100, and then both change on the server.
|
|
it("fetches the files of one changed collection and leaves the others to the next refresh", async () => {
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100), collection(2, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
client.filesFor(2, {
|
|
files: [file(2001, 2, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: SLOW_INTERVAL,
|
|
});
|
|
try {
|
|
client.filesCalls.length = 0;
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 110)],
|
|
deleted: [],
|
|
cursor: 110,
|
|
});
|
|
client.filesFor(2, {
|
|
files: [file(2002, 2, 120)],
|
|
deleted: [],
|
|
cursor: 120,
|
|
});
|
|
|
|
// Collection 1's files alone, since the stored copy.
|
|
expect(await lib.refreshCollection(collection(1, 110))).toBe(true);
|
|
expect(client.filesCalls).toEqual([
|
|
{ collectionID: 1, sinceTime: 100 },
|
|
]);
|
|
expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001, 1002]);
|
|
expect(lib.listFiles(2).map((f) => f.id)).toEqual([2001]);
|
|
|
|
// The stored copy is now current, so no request.
|
|
await lib.refreshCollection(collection(1, 110));
|
|
expect(client.filesCalls).toHaveLength(1);
|
|
|
|
// The next refresh asks from the stored cursor, 100, so it still
|
|
// sees collection 2 changed and fetches its files.
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 110), collection(2, 120)],
|
|
deleted: [],
|
|
cursor: 120,
|
|
});
|
|
await lib.fresh();
|
|
expect(client.collectionsSinceTimes).toEqual([0, 100]);
|
|
expect(client.filesCalls).toEqual([
|
|
{ collectionID: 1, sinceTime: 100 },
|
|
{ collectionID: 2, sinceTime: 100 },
|
|
]);
|
|
expect(lib.listFiles(2).map((f) => f.id)).toEqual([2001, 2002]);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// Collection 1 is stored at 100. `refreshCollection()` asks for it at 110,
|
|
// which adds file 1002. While that files request waits, a fresh read
|
|
// starts a refresh. The server tells that refresh that collection 1 was
|
|
// deleted at 130.
|
|
it("holds off a refresh that starts during the files request, which then deletes the collection", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
const files = holdNextFilesRequest(client);
|
|
try {
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 110)],
|
|
deleted: [],
|
|
cursor: 110,
|
|
});
|
|
client.collectionsQueue.push({
|
|
collections: [],
|
|
deleted: [1],
|
|
cursor: 130,
|
|
});
|
|
|
|
const refreshing = lib.refreshCollection(collection(1, 110));
|
|
await files.started;
|
|
const reading = lib.fresh();
|
|
// The refresh has not sent its collections request yet.
|
|
await new Promise((r) => setImmediate(r));
|
|
expect(client.collectionsSinceTimes).toEqual([0]);
|
|
|
|
files.release();
|
|
await refreshing;
|
|
await reading;
|
|
expect(client.collectionsSinceTimes).toEqual([0, 100]);
|
|
expect(lib.getCollection(1)).toBeUndefined();
|
|
expect(lib.listFiles(1)).toEqual([]);
|
|
} finally {
|
|
files.release();
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// `freshCollections()` lists collection 1 at 110. Before
|
|
// `refreshCollection()` is called, a refresh learns that it was deleted at
|
|
// 130.
|
|
it("stores nothing for a collection that a refresh deleted before the call", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
try {
|
|
client.collectionsQueue.push(
|
|
{ collections: [collection(1, 110)], deleted: [], cursor: 110 },
|
|
{ collections: [], deleted: [1], cursor: 130 },
|
|
);
|
|
const [listed] = await lib.freshCollections();
|
|
await lib.fresh();
|
|
client.filesCalls.length = 0;
|
|
|
|
expect(await lib.refreshCollection(listed)).toBe(false);
|
|
expect(client.filesCalls).toEqual([]);
|
|
expect(lib.getCollection(1)).toBeUndefined();
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// A refresh has stored collection 1. Collection 2 is new on the server
|
|
// since then. Only the next refresh may add it, after collection 1.
|
|
it("stores nothing for a collection that is new since the last refresh", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
try {
|
|
client.filesCalls.length = 0;
|
|
|
|
expect(await lib.refreshCollection(collection(2, 110))).toBe(false);
|
|
expect(client.filesCalls).toEqual([]);
|
|
expect(lib.getCollection(2)).toBeUndefined();
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// The caller may remove the cache directory once `close()` resolves, so
|
|
// the cache write must have landed by then.
|
|
it("close() waits for a refreshCollection() in flight", async () => {
|
|
const { client, lib } = await openSeeded(cacheDirectory);
|
|
const files = holdNextFilesRequest(client);
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 110)],
|
|
deleted: [],
|
|
cursor: 110,
|
|
});
|
|
|
|
const refreshing = lib.refreshCollection(collection(1, 110));
|
|
await files.started;
|
|
let closed = false;
|
|
const closing = lib.close().then(() => {
|
|
closed = true;
|
|
});
|
|
await new Promise((r) => setImmediate(r));
|
|
expect(closed).toBe(false);
|
|
|
|
files.release();
|
|
await closing;
|
|
const saved = await MetadataStore.load(
|
|
join(cacheDirectory, "metadata.json"),
|
|
);
|
|
expect(saved.listFiles(1).map((f) => f.id)).toEqual([1001, 1002]);
|
|
await refreshing;
|
|
});
|
|
|
|
// `quak files 2` on an empty cache stores collection 2 alone. A library
|
|
// opened later on that cache must not serve it as the whole account. Its
|
|
// first refresh replaces what the cache holds, in the server's order.
|
|
it("open() awaits the first refresh on a cache that only refreshCollection() wrote", async () => {
|
|
const client = new MockClient();
|
|
client.filesFor(2, {
|
|
files: [file(2001, 2, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
const first = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshOnOpen: false,
|
|
refreshIntervalSeconds: SLOW_INTERVAL,
|
|
});
|
|
expect(await first.refreshCollection(collection(2, 100))).toBe(true);
|
|
await first.close();
|
|
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100), collection(2, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
client.filesFor(2, {
|
|
files: [file(2001, 2, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
client.filesCalls.length = 0;
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: SLOW_INTERVAL,
|
|
});
|
|
try {
|
|
expect(lib.status().lastRefreshAt).toBeDefined();
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([1, 2]);
|
|
// It fetches collection 2's files again, from the beginning.
|
|
expect(client.filesCalls).toEqual([
|
|
{ collectionID: 1, sinceTime: 0 },
|
|
{ collectionID: 2, sinceTime: 0 },
|
|
]);
|
|
expect(lib.listFiles(2).map((f) => f.id)).toEqual([2001]);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
});
|