Resumable, deletion-aware collection and file enumeration (closes #38)
check / check (push) Successful in 24s
check / check (push) Successful in 24s
Add collectionsSince/filesSince on Client: they take a starting cursor, decrypt live records, surface tombstoned ids in a separate `deleted` list, and return the max updationTime seen as the cursor to resume from. A tombstone has no key or metadata to decrypt, so it is a bare id rather than a hollowed-out record that strict tsc would reject. filesSince paginates the diff from the given cursor and, when the server reports hasMore but the page's max updationTime does not exceed the cursor it was fetched with, throws instead of looping forever. listCollections/listFiles become thin wrappers that enumerate from sinceTime 0 and drop deletions, so existing callers and tests are unaffected. Closes #7. Model: opus-4-8
This commit is contained in:
@@ -0,0 +1,367 @@
|
||||
/**
|
||||
* Tests for the resumable, deletion-aware enumeration variants on `Client`:
|
||||
* `collectionsSince` and `filesSince`.
|
||||
*
|
||||
* The whole-account methods `listCollections` / `listFiles` always start at
|
||||
* `sinceTime: 0` and hide deletions. The cache refresh needs the opposite:
|
||||
* start from a saved cursor, learn what was deleted, and get back a cursor to
|
||||
* resume from next time. These two methods provide that.
|
||||
*
|
||||
* The return shape keeps live records and tombstones apart — `collections` /
|
||||
* `files` are decrypted live records, `deleted` is a plain list of the ids the
|
||||
* server tombstoned. A tombstone carries no decryptable key or metadata, so it
|
||||
* is a bare id rather than a hollowed-out `Collection` / `EnteFile`.
|
||||
*
|
||||
* All tests inject a fake `fetch` and drive a real `Client` (built with
|
||||
* `Client.fromJSON`) so the decryption path runs for real. Live rows are built
|
||||
* with libsodium exactly as the server would encrypt them; tombstone rows carry
|
||||
* only the fields the code reads (`id`, `updationTime`, `isDeleted`), because
|
||||
* they are never decrypted.
|
||||
*/
|
||||
|
||||
import sodium from "libsodium-wrappers-sumo";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import { init, toBase64 } from "../../src/crypto/index.js";
|
||||
import { Client, type ClientSnapshot } from "../../src/client.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixtures
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const USER_ID = 42;
|
||||
|
||||
interface Keys {
|
||||
masterKey: Uint8Array;
|
||||
publicKey: Uint8Array;
|
||||
secretKey: Uint8Array;
|
||||
}
|
||||
|
||||
const buildKeys = (): Keys => {
|
||||
const kp = sodium.crypto_box_keypair();
|
||||
return {
|
||||
masterKey: sodium.crypto_secretbox_keygen(),
|
||||
publicKey: kp.publicKey,
|
||||
secretKey: kp.privateKey,
|
||||
};
|
||||
};
|
||||
|
||||
const snapshotFor = (keys: Keys): ClientSnapshot => ({
|
||||
email: "user@example.com",
|
||||
userID: USER_ID,
|
||||
token: "test-token",
|
||||
masterKey: toBase64(keys.masterKey),
|
||||
secretKey: toBase64(keys.secretKey),
|
||||
publicKey: toBase64(keys.publicKey),
|
||||
});
|
||||
|
||||
const secretboxEncrypt = (
|
||||
plaintext: Uint8Array,
|
||||
key: Uint8Array,
|
||||
): { ciphertext: Uint8Array; nonce: Uint8Array } => {
|
||||
const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
||||
return {
|
||||
ciphertext: sodium.crypto_secretbox_easy(plaintext, nonce, key),
|
||||
nonce,
|
||||
};
|
||||
};
|
||||
|
||||
/** An owned collection row as the server sends it, keyed under the master key. */
|
||||
const ownedCollectionRow = (
|
||||
masterKey: Uint8Array,
|
||||
opts: { id: number; name: string; updationTime: number },
|
||||
): Record<string, unknown> => {
|
||||
const collectionKey = sodium.crypto_secretbox_keygen();
|
||||
const { ciphertext: encKey, nonce: keyNonce } = secretboxEncrypt(
|
||||
collectionKey,
|
||||
masterKey,
|
||||
);
|
||||
const { ciphertext: encName, nonce: nameNonce } = secretboxEncrypt(
|
||||
new TextEncoder().encode(opts.name),
|
||||
collectionKey,
|
||||
);
|
||||
return {
|
||||
id: opts.id,
|
||||
owner: { id: USER_ID },
|
||||
encryptedKey: toBase64(encKey),
|
||||
keyDecryptionNonce: toBase64(keyNonce),
|
||||
encryptedName: toBase64(encName),
|
||||
nameDecryptionNonce: toBase64(nameNonce),
|
||||
type: "album",
|
||||
updationTime: opts.updationTime,
|
||||
};
|
||||
};
|
||||
|
||||
/** A live file row inside a collection, keyed under that collection's key. */
|
||||
const fileRow = (
|
||||
collectionKey: Uint8Array,
|
||||
opts: { id: number; title: string; updationTime: number },
|
||||
): Record<string, unknown> => {
|
||||
const fileKey = sodium.crypto_secretbox_keygen();
|
||||
const { ciphertext: encFileKey, nonce: fileKeyNonce } = secretboxEncrypt(
|
||||
fileKey,
|
||||
collectionKey,
|
||||
);
|
||||
const metadata = {
|
||||
title: opts.title,
|
||||
fileType: 0,
|
||||
creationTime: opts.updationTime,
|
||||
modificationTime: opts.updationTime,
|
||||
};
|
||||
const push =
|
||||
sodium.crypto_secretstream_xchacha20poly1305_init_push(fileKey);
|
||||
const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push(
|
||||
push.state,
|
||||
new TextEncoder().encode(JSON.stringify(metadata)),
|
||||
null,
|
||||
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
||||
);
|
||||
return {
|
||||
id: opts.id,
|
||||
collectionID: 1,
|
||||
ownerID: USER_ID,
|
||||
encryptedKey: toBase64(encFileKey),
|
||||
keyDecryptionNonce: toBase64(fileKeyNonce),
|
||||
metadata: {
|
||||
encryptedData: toBase64(encMeta),
|
||||
decryptionHeader: toBase64(push.header),
|
||||
},
|
||||
file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
|
||||
thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) },
|
||||
updationTime: opts.updationTime,
|
||||
};
|
||||
};
|
||||
|
||||
/** A tombstone row. Never decrypted, so only these fields are ever read. */
|
||||
const tombstoneRow = (
|
||||
id: number,
|
||||
updationTime: number,
|
||||
): Record<string, unknown> => ({
|
||||
id,
|
||||
updationTime,
|
||||
isDeleted: true,
|
||||
});
|
||||
|
||||
const jsonResponse = (body: unknown): Response =>
|
||||
new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
|
||||
/**
|
||||
* A fetch that serves canned responses in order and records the `sinceTime`
|
||||
* query parameter each request carried, so tests can prove the cursor is
|
||||
* threaded from one page (and one call) to the next.
|
||||
*/
|
||||
const recordingFetch = (
|
||||
...responses: Response[]
|
||||
): { fetch: typeof globalThis.fetch; sinceTimes: (string | null)[] } => {
|
||||
const sinceTimes: (string | null)[] = [];
|
||||
let i = 0;
|
||||
const fake = async (input: RequestInfo | URL): Promise<Response> => {
|
||||
const url =
|
||||
typeof input === "string"
|
||||
? input
|
||||
: input instanceof URL
|
||||
? input.href
|
||||
: input.url;
|
||||
sinceTimes.push(new URL(url).searchParams.get("sinceTime"));
|
||||
if (i >= responses.length) {
|
||||
throw new Error(`recordingFetch: no response for call #${i}`);
|
||||
}
|
||||
return responses[i++]!;
|
||||
};
|
||||
return { fetch: fake as typeof globalThis.fetch, sinceTimes };
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("Client.filesSince", () => {
|
||||
beforeAll(async () => {
|
||||
await init();
|
||||
await sodium.ready;
|
||||
});
|
||||
|
||||
it("pages from the given cursor, decrypts live rows, and collects tombstones", async () => {
|
||||
const keys = buildKeys();
|
||||
const collectionKey = sodium.crypto_secretbox_keygen();
|
||||
|
||||
// Page 1 mixes a live file and a tombstone; the tombstone has the
|
||||
// higher updationTime, so it — not the live row — sets the cursor the
|
||||
// second page must be fetched from.
|
||||
const { fetch, sinceTimes } = recordingFetch(
|
||||
jsonResponse({
|
||||
diff: [
|
||||
fileRow(collectionKey, {
|
||||
id: 1001,
|
||||
title: "first.jpg",
|
||||
updationTime: 100,
|
||||
}),
|
||||
tombstoneRow(1002, 150),
|
||||
],
|
||||
hasMore: true,
|
||||
}),
|
||||
jsonResponse({
|
||||
diff: [
|
||||
fileRow(collectionKey, {
|
||||
id: 1003,
|
||||
title: "second.jpg",
|
||||
updationTime: 200,
|
||||
}),
|
||||
],
|
||||
hasMore: false,
|
||||
}),
|
||||
);
|
||||
const client = Client.fromJSON(snapshotFor(keys), { fetch });
|
||||
|
||||
const { files, deleted, cursor } = await client.filesSince({
|
||||
collectionID: 1,
|
||||
collectionKey,
|
||||
sinceTime: 0,
|
||||
});
|
||||
|
||||
expect(files.map((f) => f.id)).toEqual([1001, 1003]);
|
||||
expect(files.map((f) => f.metadata.title)).toEqual([
|
||||
"first.jpg",
|
||||
"second.jpg",
|
||||
]);
|
||||
expect(deleted).toEqual([1002]);
|
||||
expect(cursor).toBe(200);
|
||||
// First request started at the caller's cursor; the second resumed
|
||||
// from the max updationTime seen on the first page (the tombstone's).
|
||||
expect(sinceTimes).toEqual(["0", "150"]);
|
||||
});
|
||||
|
||||
it("fetches only newer rows when the returned cursor is passed back in", async () => {
|
||||
const keys = buildKeys();
|
||||
const collectionKey = sodium.crypto_secretbox_keygen();
|
||||
|
||||
const { fetch, sinceTimes } = recordingFetch(
|
||||
jsonResponse({ diff: [], hasMore: false }),
|
||||
);
|
||||
const client = Client.fromJSON(snapshotFor(keys), { fetch });
|
||||
|
||||
const result = await client.filesSince({
|
||||
collectionID: 1,
|
||||
collectionKey,
|
||||
sinceTime: 200,
|
||||
});
|
||||
|
||||
expect(result.files).toEqual([]);
|
||||
expect(result.deleted).toEqual([]);
|
||||
// An empty diff advances nothing: the cursor falls back to the input.
|
||||
expect(result.cursor).toBe(200);
|
||||
expect(sinceTimes).toEqual(["200"]);
|
||||
});
|
||||
|
||||
it("stops and throws when the server claims more but does not advance (#7)", async () => {
|
||||
const keys = buildKeys();
|
||||
const collectionKey = sodium.crypto_secretbox_keygen();
|
||||
|
||||
// hasMore is true, but the page's max updationTime (50) does not exceed
|
||||
// the cursor the request was made with (50). Following hasMore here
|
||||
// would refetch this same page forever.
|
||||
const { fetch, sinceTimes } = recordingFetch(
|
||||
jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }),
|
||||
jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }),
|
||||
);
|
||||
const client = Client.fromJSON(snapshotFor(keys), { fetch });
|
||||
|
||||
await expect(
|
||||
client.filesSince({
|
||||
collectionID: 1,
|
||||
collectionKey,
|
||||
sinceTime: 50,
|
||||
}),
|
||||
).rejects.toThrow(/not advance|non-advancing/i);
|
||||
// It gave up after the first page rather than looping.
|
||||
expect(sinceTimes).toEqual(["50"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Client.collectionsSince", () => {
|
||||
beforeAll(async () => {
|
||||
await init();
|
||||
await sodium.ready;
|
||||
});
|
||||
|
||||
it("decrypts live collections, collects tombstones, and returns a cursor", async () => {
|
||||
const keys = buildKeys();
|
||||
|
||||
const { fetch, sinceTimes } = recordingFetch(
|
||||
jsonResponse({
|
||||
collections: [
|
||||
ownedCollectionRow(keys.masterKey, {
|
||||
id: 1,
|
||||
name: "Vacation",
|
||||
updationTime: 100,
|
||||
}),
|
||||
tombstoneRow(3, 150),
|
||||
],
|
||||
}),
|
||||
);
|
||||
const client = Client.fromJSON(snapshotFor(keys), { fetch });
|
||||
|
||||
const { collections, deleted, cursor } = await client.collectionsSince({
|
||||
sinceTime: 0,
|
||||
});
|
||||
|
||||
expect(collections.map((c) => c.id)).toEqual([1]);
|
||||
expect(collections[0]!.name).toBe("Vacation");
|
||||
expect(deleted).toEqual([3]);
|
||||
// The tombstone's updationTime advances the cursor too, so the next
|
||||
// sync starts after it rather than seeing it again.
|
||||
expect(cursor).toBe(150);
|
||||
expect(sinceTimes).toEqual(["0"]);
|
||||
});
|
||||
|
||||
it("falls back to the input cursor on an empty response", async () => {
|
||||
const keys = buildKeys();
|
||||
|
||||
const { fetch, sinceTimes } = recordingFetch(
|
||||
jsonResponse({ collections: [] }),
|
||||
);
|
||||
const client = Client.fromJSON(snapshotFor(keys), { fetch });
|
||||
|
||||
const result = await client.collectionsSince({ sinceTime: 150 });
|
||||
|
||||
expect(result.collections).toEqual([]);
|
||||
expect(result.deleted).toEqual([]);
|
||||
expect(result.cursor).toBe(150);
|
||||
expect(sinceTimes).toEqual(["150"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Client list wrappers still hide deletions", () => {
|
||||
beforeAll(async () => {
|
||||
await init();
|
||||
await sodium.ready;
|
||||
});
|
||||
|
||||
it("listFiles drops tombstones and returns only live files", async () => {
|
||||
const keys = buildKeys();
|
||||
const collectionKey = sodium.crypto_secretbox_keygen();
|
||||
|
||||
const { fetch, sinceTimes } = recordingFetch(
|
||||
jsonResponse({
|
||||
diff: [
|
||||
fileRow(collectionKey, {
|
||||
id: 7,
|
||||
title: "keep.jpg",
|
||||
updationTime: 100,
|
||||
}),
|
||||
tombstoneRow(8, 150),
|
||||
],
|
||||
hasMore: false,
|
||||
}),
|
||||
);
|
||||
const client = Client.fromJSON(snapshotFor(keys), { fetch });
|
||||
|
||||
const files = await client.listFiles(1, collectionKey);
|
||||
|
||||
expect(files.map((f) => f.id)).toEqual([7]);
|
||||
// The wrapper starts a full enumeration from zero.
|
||||
expect(sinceTimes).toEqual(["0"]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user