sneak
6f8f85fff5
check / check (push) Successful in 3m29s
`quak login` takes the server's API address from `--server` or `QUAK_SERVER`. `isServerAddress` refuses anything but an http or https URL with no query or fragment, there and in `Client.fromJSON`. The client's snapshot keeps a non-default `apiOrigin`, so the session file names the server and every later command uses it. Two servers can give two accounts, and two files, the same ID. A self-hosted account's default cache directory therefore names the server, and `metadata.json` records it: `Library.open` deletes a cache from another server whole. `originalOf` and `thumbnailOf` record the account in a cache that names none before storing a file, and `open` treats a cache with no collection as empty. This fixes #254 (closes #254). Model: opus-5-5
1058 lines
38 KiB
TypeScript
1058 lines
38 KiB
TypeScript
/**
|
|
* Tests for `Library.open()` and its transparent background refresh loop.
|
|
*
|
|
* The library keeps the account's server state in a `MetadataStore` (issue
|
|
* #41) and pulls changes with the resumable, tombstone-aware enumerators on
|
|
* `Client` (issue #38: `collectionsSince` / `filesSince`). `open()` loads the
|
|
* cache, does one refresh, then refreshes again every `refreshIntervalSeconds`
|
|
* on a background timer. The design (#36) forbids an exposed `sync()`, a
|
|
* `serverReachable` flag, a `lib.refresh()` method, and a "before each read"
|
|
* mode. The contracts exercised here:
|
|
*
|
|
* 1. Reads are answered from RAM. A read never calls the client.
|
|
* 2. `open()` does an initial refresh, then the interval keeps refreshing;
|
|
* each refresh resumes from the stored cursor and applies diffs + tombstones.
|
|
* 3. The cache is rewritten only when a refresh actually changes something.
|
|
* 4. A failed refresh is invisible to reads: the last good data stays, the
|
|
* failure surfaces via `onProgress` ("failed") and `status()`, and a later
|
|
* success clears the error. `open()` itself resolves even when the first
|
|
* refresh fails (offline start from cache).
|
|
* 5. `close()` stops the timer and is idempotent, and its promise resolves
|
|
* only once an in-flight refresh has written the cache file.
|
|
* 6. `cacheDirectory` defaults to the env-paths cache dir plus the user id,
|
|
* with a self-hosted server's address before the user id.
|
|
* 7. `open()` branches on the cache: a cache that holds no collection awaits
|
|
* the first refresh (it has nothing to serve yet); an existing cache serves
|
|
* its copy at once and refreshes in the background, so a slow or dead
|
|
* server never stalls opening.
|
|
* 8. A save failure that leaves RAM ahead of disk keeps `status().lastError`
|
|
* set and keeps retrying the write; a later empty refresh does not clear it.
|
|
* 9. With `refreshOnOpen: false`, `open()` starts no refresh at all, and a
|
|
* refresh reports through `onProgress` each collection whose files it
|
|
* fetched.
|
|
* 10. A cache another account filled starts empty. One filled from another
|
|
* server also loses its originals and thumbnails, since two servers can
|
|
* give two files the same ID. `thumbnailOf` on an empty cache saves the
|
|
* account's user ID and server. A refresh saves the account's server, so a
|
|
* self-hosted account keeps its own cache from one `open()` to the next.
|
|
*
|
|
* The client is a mock: no crypto, no network. It serves scripted pages and
|
|
* records the `sinceTime` each call carried so cursor threading is provable.
|
|
*
|
|
* On an empty cache `open()` awaits the initial refresh (including its cache
|
|
* write), so state right after `open()` is deterministic; the tests that
|
|
* inspect post-`open()` state seed no cache and rely on that. Tests for an
|
|
* existing-cache open seed a store first and prove `open()` returns without
|
|
* waiting for the network. The interval tests then use real timers with a
|
|
* short interval and `vi.waitFor`: a fake clock cannot settle the real
|
|
* fsync-and-rename cache write, and empty diffs never write, so the eventual
|
|
* state is stable to poll for.
|
|
*
|
|
* A refresh changes RAM before it writes the cache file, so a polled state can
|
|
* be visible while that write is still running. Every test therefore awaits
|
|
* `close()`, which waits for the in-flight refresh, before `afterEach` removes
|
|
* the directory.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
|
import {
|
|
existsSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import envPaths from "env-paths";
|
|
|
|
import { Library, type RefreshEvent } from "../../src/library/index.js";
|
|
import { MetadataStore } from "../../src/library/store.js";
|
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
|
|
|
const USER_ID = 42;
|
|
|
|
// Short enough that a couple of ticks pass within a test, long enough not to
|
|
// spin; interval tests poll for the eventual state rather than counting ticks.
|
|
const FAST_INTERVAL = 0.02;
|
|
|
|
const collection = (
|
|
id: number,
|
|
updationTime: number,
|
|
name = `album-${id}`,
|
|
): Collection => ({
|
|
id,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
name,
|
|
type: "album",
|
|
updationTime,
|
|
isShared: false,
|
|
});
|
|
|
|
const file = (
|
|
id: number,
|
|
collectionID: number,
|
|
updationTime: number,
|
|
): EnteFile => ({
|
|
id,
|
|
collectionID,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
metadata: {
|
|
title: `file-${id}.jpg`,
|
|
fileType: "image",
|
|
creationTime: updationTime,
|
|
modificationTime: updationTime,
|
|
},
|
|
file: { decryptionHeader: "aGVhZGVy" },
|
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
|
updationTime,
|
|
});
|
|
|
|
/**
|
|
* A mock `Client`. `collectionsSince` shifts one page off `collectionsQueue`
|
|
* per call (an empty diff that advances nothing when the queue runs dry);
|
|
* `filesSince` shifts from a per-collection queue. `failCollections` makes the
|
|
* next and all further collection fetches throw, to simulate an offline server.
|
|
*/
|
|
class MockClient {
|
|
userID = USER_ID;
|
|
apiOrigin?: string;
|
|
failCollections = false;
|
|
collectionsQueue: CollectionsPage[] = [];
|
|
filesByCollection = new Map<number, FilesPage[]>();
|
|
|
|
collectionsSinceTimes: number[] = [];
|
|
filesCalls: { collectionID: number; sinceTime: number }[] = [];
|
|
|
|
whoami(): { email: string; userID: number } {
|
|
return { email: "user@example.com", userID: this.userID };
|
|
}
|
|
|
|
async collectionsSince(args: {
|
|
sinceTime: number;
|
|
}): Promise<CollectionsPage> {
|
|
this.collectionsSinceTimes.push(args.sinceTime);
|
|
if (this.failCollections) throw new Error("network down");
|
|
return (
|
|
this.collectionsQueue.shift() ?? {
|
|
collections: [],
|
|
deleted: [],
|
|
cursor: args.sinceTime,
|
|
}
|
|
);
|
|
}
|
|
|
|
async filesSince(args: {
|
|
collectionID: number;
|
|
collectionKey: Uint8Array;
|
|
sinceTime: number;
|
|
}): Promise<FilesPage> {
|
|
this.filesCalls.push({
|
|
collectionID: args.collectionID,
|
|
sinceTime: args.sinceTime,
|
|
});
|
|
const queue = this.filesByCollection.get(args.collectionID);
|
|
return (
|
|
queue?.shift() ?? {
|
|
files: [],
|
|
deleted: [],
|
|
cursor: args.sinceTime,
|
|
}
|
|
);
|
|
}
|
|
|
|
filesFor(collectionID: number, ...pages: FilesPage[]): void {
|
|
this.filesByCollection.set(collectionID, pages);
|
|
}
|
|
}
|
|
|
|
describe("Library.open and background refresh", () => {
|
|
let dir: string;
|
|
let cacheDirectory: string;
|
|
|
|
beforeEach(() => {
|
|
dir = mkdtempSync(join(tmpdir(), "quak-library-"));
|
|
cacheDirectory = join(dir, "cache");
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("does an initial refresh and answers reads from the cache", async () => {
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90), file(1002, 1, 95)],
|
|
deleted: [],
|
|
cursor: 95,
|
|
});
|
|
|
|
const lib = await Library.open({ client, cacheDirectory });
|
|
try {
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([1]);
|
|
expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001, 1002]);
|
|
expect(lib.getFile(1, 1001)?.metadata.title).toBe("file-1001.jpg");
|
|
|
|
const status = lib.status();
|
|
expect(status.userID).toBe(USER_ID);
|
|
expect(status.collections).toBe(1);
|
|
expect(status.files).toBe(2);
|
|
expect(status.lastRefreshAt).toBeGreaterThan(0);
|
|
expect(status.lastError).toBeUndefined();
|
|
|
|
// The initial refresh persisted the cache to disk.
|
|
const reloaded = await MetadataStore.load(
|
|
join(cacheDirectory, "metadata.json"),
|
|
);
|
|
expect(reloaded.getFile(1, 1001)?.id).toBe(1001);
|
|
expect(reloaded.collectionsSinceTime).toBe(100);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("reads never call the client", async () => {
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
|
|
const lib = await Library.open({ client, cacheDirectory });
|
|
try {
|
|
const collectionCalls = client.collectionsSinceTimes.length;
|
|
const fileCalls = client.filesCalls.length;
|
|
|
|
lib.listCollections();
|
|
lib.getCollection(1);
|
|
lib.listFiles(1);
|
|
lib.getFile(1, 1001);
|
|
lib.status();
|
|
|
|
expect(client.collectionsSinceTimes.length).toBe(collectionCalls);
|
|
expect(client.filesCalls.length).toBe(fileCalls);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("resumes each refresh from the stored cursor", async () => {
|
|
// Seed a cache with a cursor and a collection, as a prior run left it.
|
|
const path = join(cacheDirectory, "metadata.json");
|
|
const seed = await MetadataStore.load(path);
|
|
seed.userID = USER_ID;
|
|
seed.collectionsSinceTime = 500;
|
|
seed.putCollection(collection(1, 400));
|
|
seed.putFile(file(1001, 1, 400));
|
|
await seed.save();
|
|
|
|
const client = new MockClient();
|
|
// The collection's updationTime advances (400 -> 600), so its files are
|
|
// re-enumerated from the collection's stored updationTime (400).
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 600)],
|
|
deleted: [],
|
|
cursor: 600,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 550)],
|
|
deleted: [],
|
|
cursor: 550,
|
|
});
|
|
|
|
// Opening from an existing cache serves the seeded copy at once and
|
|
// refreshes in the background, so the refresh's effects are polled for.
|
|
const lib = await Library.open({ client, cacheDirectory });
|
|
try {
|
|
await vi.waitFor(
|
|
() => {
|
|
// Collections resumed from the stored cursor, and files were
|
|
// re-enumerated from the stored collection updationTime.
|
|
expect(client.collectionsSinceTimes[0]).toBe(500);
|
|
expect(client.filesCalls).toEqual([
|
|
{ collectionID: 1, sinceTime: 400 },
|
|
]);
|
|
expect(lib.listFiles(1).map((f) => f.id)).toEqual([
|
|
1001, 1002,
|
|
]);
|
|
},
|
|
{ timeout: 2000, interval: 5 },
|
|
);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("does not re-enumerate a collection whose updationTime did not advance", async () => {
|
|
const path = join(cacheDirectory, "metadata.json");
|
|
const seed = await MetadataStore.load(path);
|
|
seed.userID = USER_ID;
|
|
seed.collectionsSinceTime = 100;
|
|
seed.putCollection(collection(1, 400));
|
|
await seed.save();
|
|
|
|
const client = new MockClient();
|
|
// The collection comes back in the diff (its metadata changed) but at
|
|
// the same updationTime, so its files must not be re-fetched.
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 400, "renamed")],
|
|
deleted: [],
|
|
cursor: 400,
|
|
});
|
|
|
|
// Existing cache: the rename lands via the background refresh.
|
|
const lib = await Library.open({ client, cacheDirectory });
|
|
try {
|
|
await vi.waitFor(
|
|
() => expect(lib.getCollection(1)?.name).toBe("renamed"),
|
|
{ timeout: 2000, interval: 5 },
|
|
);
|
|
// The collection's updationTime did not advance, so its files were
|
|
// never re-fetched.
|
|
expect(client.filesCalls).toEqual([]);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("applies diffs and tombstones on the interval", async () => {
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100), collection(2, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
client.filesFor(2, {
|
|
files: [file(2001, 2, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: FAST_INTERVAL,
|
|
});
|
|
try {
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([1, 2]);
|
|
expect(lib.listFiles(2).map((f) => f.id)).toEqual([2001]);
|
|
|
|
// Next refresh: collection 2 is tombstoned; collection 1 gains a
|
|
// file and loses its old one.
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 190)],
|
|
deleted: [1001],
|
|
cursor: 190,
|
|
});
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 200)],
|
|
deleted: [2],
|
|
cursor: 200,
|
|
});
|
|
|
|
await vi.waitFor(
|
|
() => {
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([1]);
|
|
expect(lib.listFiles(1).map((f) => f.id)).toEqual([1002]);
|
|
// Collection 2's files went with it.
|
|
expect(lib.listFiles(2)).toEqual([]);
|
|
},
|
|
{ timeout: 2000, interval: 5 },
|
|
);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("rewrites the cache only when a refresh changes something", async () => {
|
|
const saveSpy = vi.spyOn(MetadataStore.prototype, "save");
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: FAST_INTERVAL,
|
|
});
|
|
try {
|
|
// The initial refresh changed everything, so it saved once.
|
|
expect(saveSpy).toHaveBeenCalledTimes(1);
|
|
|
|
// Several empty-diff ticks pass; none of them may rewrite the file.
|
|
await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 4));
|
|
expect(saveSpy).toHaveBeenCalledTimes(1);
|
|
|
|
// A real change triggers exactly one more rewrite; later empty ticks
|
|
// still do not, so the count settles at two.
|
|
client.collectionsQueue.push({
|
|
collections: [collection(2, 200)],
|
|
deleted: [],
|
|
cursor: 200,
|
|
});
|
|
await vi.waitFor(() => expect(saveSpy).toHaveBeenCalledTimes(2), {
|
|
timeout: 2000,
|
|
interval: 5,
|
|
});
|
|
await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 4));
|
|
expect(saveSpy).toHaveBeenCalledTimes(2);
|
|
} finally {
|
|
await lib.close();
|
|
saveSpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("keeps a failed refresh invisible to reads and recovers later", async () => {
|
|
const events: RefreshEvent[] = [];
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: FAST_INTERVAL,
|
|
onProgress: (e) => events.push(e),
|
|
});
|
|
try {
|
|
expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]);
|
|
|
|
// The server goes away; refreshes now fail.
|
|
client.failCollections = true;
|
|
await vi.waitFor(
|
|
() => expect(lib.status().lastError).toMatch(/network down/),
|
|
{ timeout: 2000, interval: 5 },
|
|
);
|
|
|
|
// Reads still see the last good data; the failure was reported.
|
|
expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]);
|
|
expect(
|
|
events.some(
|
|
(e) => e.operation === "refresh" && e.status === "failed",
|
|
),
|
|
).toBe(true);
|
|
|
|
// Recovery: a later refresh succeeds and clears the error.
|
|
client.failCollections = false;
|
|
client.collectionsQueue.push({
|
|
collections: [collection(2, 300)],
|
|
deleted: [],
|
|
cursor: 300,
|
|
});
|
|
await vi.waitFor(
|
|
() => {
|
|
expect(lib.status().lastError).toBeUndefined();
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([
|
|
1, 2,
|
|
]);
|
|
},
|
|
{ timeout: 2000, interval: 5 },
|
|
);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("resolves open() even when the first refresh fails", async () => {
|
|
const client = new MockClient();
|
|
client.failCollections = true;
|
|
const events: RefreshEvent[] = [];
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
onProgress: (e) => events.push(e),
|
|
});
|
|
try {
|
|
// Nothing was cached and the server is unreachable: reads are empty,
|
|
// but the library opened and the failure is on record.
|
|
expect(lib.listCollections()).toEqual([]);
|
|
expect(lib.status().lastError).toMatch(/network down/);
|
|
expect(lib.status().lastRefreshAt).toBeUndefined();
|
|
expect(
|
|
events.some(
|
|
(e) => e.operation === "refresh" && e.status === "failed",
|
|
),
|
|
).toBe(true);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("opens from an existing cache without waiting for the first refresh", async () => {
|
|
// Seed a cache as a prior run left it.
|
|
const path = join(cacheDirectory, "metadata.json");
|
|
const seed = await MetadataStore.load(path);
|
|
seed.userID = USER_ID;
|
|
seed.collectionsSinceTime = 500;
|
|
seed.putCollection(collection(1, 400));
|
|
seed.putFile(file(1001, 1, 400));
|
|
await seed.save();
|
|
|
|
// The server does not answer this run's first refresh until the test
|
|
// is done with it.
|
|
let answerFirstFetch: (page: CollectionsPage) => void = () => {};
|
|
const client = new MockClient();
|
|
client.collectionsSince = () =>
|
|
new Promise<CollectionsPage>((resolve) => {
|
|
answerFirstFetch = resolve;
|
|
});
|
|
|
|
// open() must resolve from the cache without blocking on the network,
|
|
// and reads must serve the seeded copy.
|
|
const lib = await Library.open({ client, cacheDirectory });
|
|
try {
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([1]);
|
|
expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]);
|
|
// The first refresh is still outstanding: nothing has completed or
|
|
// failed yet.
|
|
expect(lib.status().lastRefreshAt).toBeUndefined();
|
|
expect(lib.status().lastError).toBeUndefined();
|
|
} finally {
|
|
// close() waits for the outstanding refresh, so let it finish.
|
|
answerFirstFetch({ collections: [], deleted: [], cursor: 500 });
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("awaits the first refresh on a first run with an empty cache", async () => {
|
|
// No cache on disk: open() must not resolve until the first fetch does,
|
|
// so it never hands back an empty library it could have filled.
|
|
let releaseFirstFetch: (page: CollectionsPage) => void = () => {};
|
|
const gate = new Promise<CollectionsPage>((resolve) => {
|
|
releaseFirstFetch = resolve;
|
|
});
|
|
const client = new MockClient();
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
client.collectionsSince = async (args: { sinceTime: number }) => {
|
|
client.collectionsSinceTimes.push(args.sinceTime);
|
|
return gate;
|
|
};
|
|
|
|
let opened = false;
|
|
const openPromise = Library.open({ client, cacheDirectory }).then(
|
|
(l) => {
|
|
opened = true;
|
|
return l;
|
|
},
|
|
);
|
|
|
|
// While the first fetch is outstanding, open() has not resolved.
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
expect(opened).toBe(false);
|
|
|
|
// Completing the fetch lets open() resolve with the data in place.
|
|
releaseFirstFetch({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
const lib = await openPromise;
|
|
try {
|
|
expect(opened).toBe(true);
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([1]);
|
|
expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]);
|
|
expect(lib.status().lastRefreshAt).toBeGreaterThan(0);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// `originalOf` on an empty cache leaves such a cache behind: it names the
|
|
// account before it stores the original.
|
|
it("awaits the first refresh on a cache that names the account but holds no collection", async () => {
|
|
const seed = await MetadataStore.load(
|
|
join(cacheDirectory, "metadata.json"),
|
|
);
|
|
seed.userID = USER_ID;
|
|
await seed.save();
|
|
let releaseFirstFetch: (page: CollectionsPage) => void = () => {};
|
|
const gate = new Promise<CollectionsPage>((resolve) => {
|
|
releaseFirstFetch = resolve;
|
|
});
|
|
const client = new MockClient();
|
|
client.collectionsSince = async () => gate;
|
|
|
|
let opened = false;
|
|
const openPromise = Library.open({ client, cacheDirectory }).then(
|
|
(l) => {
|
|
opened = true;
|
|
return l;
|
|
},
|
|
);
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
expect(opened).toBe(false);
|
|
|
|
releaseFirstFetch({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
const lib = await openPromise;
|
|
try {
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([1]);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// A caller that asks for each refresh it needs, as the CLI's read commands
|
|
// do, opens without one: on an empty cache the refresh `open()` would
|
|
// await fetches the files of every collection, which `quak collections`
|
|
// never needs (issue #210).
|
|
it("starts no refresh as it opens when refreshOnOpen is false", async () => {
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshOnOpen: false,
|
|
refreshIntervalSeconds: Infinity,
|
|
});
|
|
try {
|
|
expect(client.collectionsSinceTimes).toEqual([]);
|
|
expect(lib.listCollections()).toEqual([]);
|
|
expect(lib.status().lastRefreshAt).toBeUndefined();
|
|
|
|
// The refresh comes when the caller asks for it.
|
|
await lib.fresh();
|
|
expect(client.collectionsSinceTimes).toEqual([0]);
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([1]);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// A refresh fetches the files of each collection that changed, one
|
|
// collection after another, and stores nothing until it has them all. On
|
|
// a large account that takes a while, so it reports each collection whose
|
|
// files it has fetched, and how many it will fetch in all (issue #210).
|
|
it("reports each collection whose files a refresh fetched", async () => {
|
|
const events: RefreshEvent[] = [];
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [
|
|
collection(1, 100),
|
|
collection(2, 100),
|
|
collection(3, 100),
|
|
],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: Infinity,
|
|
onProgress: (e) => events.push(e),
|
|
});
|
|
try {
|
|
expect(events).toEqual([
|
|
{ operation: "refresh", status: "started" },
|
|
{
|
|
operation: "refresh",
|
|
status: "progress",
|
|
fetched: 1,
|
|
total: 3,
|
|
},
|
|
{
|
|
operation: "refresh",
|
|
status: "progress",
|
|
fetched: 2,
|
|
total: 3,
|
|
},
|
|
{
|
|
operation: "refresh",
|
|
status: "progress",
|
|
fetched: 3,
|
|
total: 3,
|
|
},
|
|
{ operation: "refresh", status: "done" },
|
|
]);
|
|
|
|
// Only collection 2 changed since, so the next refresh fetches
|
|
// its files alone and counts one.
|
|
events.length = 0;
|
|
client.collectionsQueue.push({
|
|
collections: [collection(2, 200)],
|
|
deleted: [],
|
|
cursor: 200,
|
|
});
|
|
await lib.fresh();
|
|
expect(events.filter((e) => e.status === "progress")).toEqual([
|
|
{
|
|
operation: "refresh",
|
|
status: "progress",
|
|
fetched: 1,
|
|
total: 1,
|
|
},
|
|
]);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
it("keeps a save failure visible until a save actually succeeds", async () => {
|
|
const saveSpy = vi
|
|
.spyOn(MetadataStore.prototype, "save")
|
|
.mockRejectedValue(new Error("disk full"));
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1001, 1, 90)],
|
|
deleted: [],
|
|
cursor: 90,
|
|
});
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: FAST_INTERVAL,
|
|
});
|
|
try {
|
|
// The initial refresh mutated RAM but its save failed, so the error
|
|
// is on record and no refresh has counted as successful.
|
|
expect(lib.status().lastError).toMatch(/disk full/);
|
|
expect(lib.status().lastRefreshAt).toBeUndefined();
|
|
|
|
// Empty-diff ticks pass. Each still retries the unsaved write and
|
|
// still fails, so the error never silently clears and the refresh
|
|
// clock never advances — RAM must not run ahead of disk unnoticed.
|
|
const savesBefore = saveSpy.mock.calls.length;
|
|
await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 4));
|
|
expect(saveSpy.mock.calls.length).toBeGreaterThan(savesBefore);
|
|
expect(lib.status().lastError).toMatch(/disk full/);
|
|
expect(lib.status().lastRefreshAt).toBeUndefined();
|
|
|
|
// Once the disk recovers, the next tick persists the pending change
|
|
// and only then clears the error and advances the clock.
|
|
saveSpy.mockRestore();
|
|
await vi.waitFor(
|
|
() => {
|
|
expect(lib.status().lastError).toBeUndefined();
|
|
expect(lib.status().lastRefreshAt).toBeGreaterThan(0);
|
|
},
|
|
{ timeout: 2000, interval: 5 },
|
|
);
|
|
const reloaded = await MetadataStore.load(
|
|
join(cacheDirectory, "metadata.json"),
|
|
);
|
|
expect(reloaded.getFile(1, 1001)?.id).toBe(1001);
|
|
} finally {
|
|
await lib.close();
|
|
saveSpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("close() stops the timer and is idempotent", async () => {
|
|
const client = new MockClient();
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: FAST_INTERVAL,
|
|
});
|
|
const callsAfterOpen = client.collectionsSinceTimes.length;
|
|
|
|
await lib.close();
|
|
await lib.close(); // second close must not throw
|
|
expect(lib.status().closed).toBe(true);
|
|
|
|
// No further refreshes fire once closed.
|
|
await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 5));
|
|
expect(client.collectionsSinceTimes.length).toBe(callsAfterOpen);
|
|
});
|
|
|
|
it("close() resolves only after an in-flight refresh has written the cache", async () => {
|
|
const path = join(cacheDirectory, "metadata.json");
|
|
const seed = await MetadataStore.load(path);
|
|
seed.userID = USER_ID;
|
|
seed.collectionsSinceTime = 500;
|
|
seed.putCollection(collection(1, 400));
|
|
seed.putFile(file(1001, 1, 400));
|
|
await seed.save();
|
|
|
|
const client = new MockClient();
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 600)],
|
|
deleted: [],
|
|
cursor: 600,
|
|
});
|
|
client.filesFor(1, {
|
|
files: [file(1002, 1, 600)],
|
|
deleted: [],
|
|
cursor: 600,
|
|
});
|
|
|
|
// Hold the refresh's cache write until the test releases it.
|
|
const realSave = MetadataStore.prototype.save;
|
|
let releaseSave: () => void = () => {};
|
|
const saveHeld = new Promise<void>((resolve) => {
|
|
releaseSave = resolve;
|
|
});
|
|
const saveSpy = vi
|
|
.spyOn(MetadataStore.prototype, "save")
|
|
.mockImplementation(async function (this: MetadataStore) {
|
|
await saveHeld;
|
|
return realSave.call(this);
|
|
});
|
|
|
|
const lib = await Library.open({ client, cacheDirectory });
|
|
try {
|
|
await vi.waitFor(() => expect(saveSpy).toHaveBeenCalled(), {
|
|
timeout: 2000,
|
|
interval: 5,
|
|
});
|
|
|
|
let closed = false;
|
|
const closing = lib.close().then(() => {
|
|
closed = true;
|
|
});
|
|
await new Promise((r) => setTimeout(r, 50));
|
|
expect(closed).toBe(false);
|
|
|
|
releaseSave();
|
|
await closing;
|
|
const reloaded = await MetadataStore.load(path);
|
|
expect(reloaded.getFile(1, 1002)?.id).toBe(1002);
|
|
} finally {
|
|
releaseSave();
|
|
await lib.close();
|
|
saveSpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("defaults cacheDirectory to the env-paths cache dir plus user id", async () => {
|
|
const xdg = join(dir, "xdg-cache");
|
|
const prev = process.env.XDG_CACHE_HOME;
|
|
process.env.XDG_CACHE_HOME = xdg;
|
|
try {
|
|
const client = new MockClient();
|
|
const lib = await Library.open({ client });
|
|
try {
|
|
const expected = join(
|
|
envPaths("quak", { suffix: "" }).cache,
|
|
String(USER_ID),
|
|
);
|
|
expect(lib.cacheDirectory).toBe(expected);
|
|
expect(lib.cacheDirectory.startsWith(xdg)).toBe(true);
|
|
expect(lib.cacheDirectory.endsWith(String(USER_ID))).toBe(true);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
} finally {
|
|
if (prev === undefined) delete process.env.XDG_CACHE_HOME;
|
|
else process.env.XDG_CACHE_HOME = prev;
|
|
}
|
|
});
|
|
|
|
it("puts a self-hosted server's address, URL-encoded, before the user id in the default cacheDirectory", async () => {
|
|
const xdg = join(dir, "xdg-cache");
|
|
const prev = process.env.XDG_CACHE_HOME;
|
|
process.env.XDG_CACHE_HOME = xdg;
|
|
try {
|
|
const client = new MockClient();
|
|
client.apiOrigin = "https://ente.example.com:8443/api";
|
|
const lib = await Library.open({ client });
|
|
try {
|
|
expect(lib.cacheDirectory).toBe(
|
|
join(
|
|
envPaths("quak", { suffix: "" }).cache,
|
|
"https%3A%2F%2Fente.example.com%3A8443%2Fapi",
|
|
String(USER_ID),
|
|
),
|
|
);
|
|
expect(lib.cacheDirectory.startsWith(xdg)).toBe(true);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
} finally {
|
|
if (prev === undefined) delete process.env.XDG_CACHE_HOME;
|
|
else process.env.XDG_CACHE_HOME = prev;
|
|
}
|
|
});
|
|
|
|
// A cache that account USER_ID on `apiOrigin` filled: its records, and an
|
|
// original and a thumbnail of file 1001.
|
|
const fillCache = async (apiOrigin?: string): Promise<void> => {
|
|
const seed = await MetadataStore.load(
|
|
join(cacheDirectory, "metadata.json"),
|
|
);
|
|
seed.userID = USER_ID;
|
|
seed.apiOrigin = apiOrigin;
|
|
seed.collectionsSinceTime = 500;
|
|
seed.putCollection(collection(1, 400));
|
|
seed.putFile(file(1001, 1, 400));
|
|
await seed.save();
|
|
for (const name of ["originals", "thumbnails"]) {
|
|
mkdirSync(join(cacheDirectory, name));
|
|
writeFileSync(join(cacheDirectory, name, "1001.jpg"), "1001");
|
|
}
|
|
};
|
|
|
|
it("deletes the whole cache an account on another server filled", async () => {
|
|
await fillCache("https://ente.example.com");
|
|
const client = new MockClient();
|
|
client.apiOrigin = "https://photos.example.org";
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: 3600,
|
|
});
|
|
try {
|
|
expect(client.collectionsSinceTimes).toEqual([0]);
|
|
expect(lib.listCollections()).toEqual([]);
|
|
for (const name of ["originals", "thumbnails"]) {
|
|
expect(existsSync(join(cacheDirectory, name, "1001.jpg"))).toBe(
|
|
false,
|
|
);
|
|
}
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// A server gives each file its own ID, so another account's cached file
|
|
// is never served for this account's.
|
|
it("keeps the originals and thumbnails another account on the same server cached", async () => {
|
|
await fillCache();
|
|
const client = new MockClient();
|
|
client.userID = USER_ID + 1;
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: 3600,
|
|
});
|
|
try {
|
|
expect(client.collectionsSinceTimes).toEqual([0]);
|
|
expect(lib.listCollections()).toEqual([]);
|
|
for (const name of ["originals", "thumbnails"]) {
|
|
expect(existsSync(join(cacheDirectory, name, "1001.jpg"))).toBe(
|
|
true,
|
|
);
|
|
}
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// `quak get-thumb` on an empty cache stores a thumbnail before any refresh
|
|
// runs. `open()` for an account on another server deletes that thumbnail
|
|
// only if the cache names the account that stored it.
|
|
it("thumbnailOf on an empty cache saves the account's user ID and server in metadata.json", async () => {
|
|
const client = new MockClient();
|
|
client.apiOrigin = "https://ente.example.com";
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshOnOpen: false,
|
|
refreshIntervalSeconds: Infinity,
|
|
contentSource: {
|
|
original: async () => {
|
|
throw new Error("original not expected");
|
|
},
|
|
thumbnail: async ({ destination }) => {
|
|
writeFileSync(destination, "thumb");
|
|
return { bytesWritten: 5 };
|
|
},
|
|
},
|
|
});
|
|
try {
|
|
await lib.thumbnailOf(file(1001, 1, 400));
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
|
|
const saved = await MetadataStore.load(
|
|
join(cacheDirectory, "metadata.json"),
|
|
);
|
|
expect(saved.userID).toBe(USER_ID);
|
|
expect(saved.apiOrigin).toBe("https://ente.example.com");
|
|
});
|
|
|
|
// A refresh saves the account's server in `metadata.json`. Without it,
|
|
// `open()` would take a self-hosted account's own cache for another
|
|
// server's and delete it.
|
|
it("keeps a self-hosted account's own cache from one open to the next", async () => {
|
|
const client = new MockClient();
|
|
client.apiOrigin = "https://ente.example.com";
|
|
client.collectionsQueue.push({
|
|
collections: [collection(1, 100)],
|
|
deleted: [],
|
|
cursor: 100,
|
|
});
|
|
const first = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: Infinity,
|
|
});
|
|
await first.close();
|
|
mkdirSync(join(cacheDirectory, "originals"));
|
|
writeFileSync(join(cacheDirectory, "originals", "1001.jpg"), "1001");
|
|
|
|
const lib = await Library.open({
|
|
client,
|
|
cacheDirectory,
|
|
refreshIntervalSeconds: Infinity,
|
|
});
|
|
try {
|
|
expect(lib.listCollections().map((c) => c.id)).toEqual([1]);
|
|
expect(
|
|
existsSync(join(cacheDirectory, "originals", "1001.jpg")),
|
|
).toBe(true);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
// The second open's refresh started from the saved cursor.
|
|
expect(client.collectionsSinceTimes).toEqual([0, 100]);
|
|
});
|
|
});
|