sneak
8e06cd5723
check / check (push) Canceled after 0s
`quak files COLLECTION` used to wait for a full refresh, which on an empty cache fetched the files of every collection before it listed one. It now reads the collection list, then calls the new `Library.refreshCollection()`, which brings that one collection up to date in the store as a refresh would, and saves it. It sends no request for the files when the stored copy is current. It leaves the refresh cursor alone, so the next refresh still fetches every other collection that changed. `refreshOnce` and `refreshCollection` share the commit step. `quak files` with no argument still waits for a refresh. Model: opus-5-5
1770 lines
65 KiB
TypeScript
1770 lines
65 KiB
TypeScript
/**
|
|
* Tests for the CLI commands (`src/cli-commands.ts`, issue #12).
|
|
*
|
|
* Each command is called directly with a context whose output streams collect
|
|
* text, whose session directory is a fresh temp directory, and whose session
|
|
* loader hands back a fake client. The fake serves two albums and three files
|
|
* from memory, writes stand-in bytes for originals and thumbnails, and makes no
|
|
* network calls. The helpers the commands call (`cli-read`, `cli-output`,
|
|
* backup, thumbnails) have their own tests; these check what each command
|
|
* prints and the exit code it returns.
|
|
*/
|
|
|
|
import {
|
|
existsSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readdirSync,
|
|
readFileSync,
|
|
rmSync,
|
|
statSync,
|
|
utimesSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { basename, join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { PassThrough } from "node:stream";
|
|
import * as jpegJs from "jpeg-js";
|
|
import sodium from "libsodium-wrappers-sumo";
|
|
import {
|
|
describe,
|
|
it,
|
|
expect,
|
|
vi,
|
|
beforeAll,
|
|
beforeEach,
|
|
afterEach,
|
|
} from "vitest";
|
|
|
|
import {
|
|
type CliContext,
|
|
saveSession,
|
|
loginCommand,
|
|
whoamiCommand,
|
|
logoutCommand,
|
|
collectionsCommand,
|
|
filesCommand,
|
|
getCommand,
|
|
getThumbCommand,
|
|
backupCommand,
|
|
backupMetadataCommand,
|
|
listMissingThumbnailsCommand,
|
|
fixMissingThumbnailsCommand,
|
|
} from "../../src/cli-commands.js";
|
|
import { stderrLog } from "../../src/cli-log.js";
|
|
import { run } from "../../src/cli-run.js";
|
|
import { loadSession } from "../../src/cli-session.js";
|
|
import {
|
|
Client,
|
|
type ClientSnapshot,
|
|
type LoginOptions,
|
|
} from "../../src/client.js";
|
|
import { savePath, type ContentSource } from "../../src/library/content.js";
|
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
|
import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js";
|
|
import { defaultCacheDirectory } from "../../src/library/index.js";
|
|
import {
|
|
buildAccount,
|
|
buildMetaFetch,
|
|
encryptSecretbox,
|
|
encryptStreamBlob,
|
|
} from "../ente-server.js";
|
|
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
|
import {
|
|
asLivePhoto,
|
|
blake2b,
|
|
cdnSource,
|
|
IMAGE,
|
|
livePhotoHash,
|
|
livePhotoZip,
|
|
VIDEO,
|
|
} from "../live-photo.js";
|
|
|
|
const USER_ID = 42;
|
|
|
|
const collection = (
|
|
id: number,
|
|
name: string,
|
|
isShared = false,
|
|
): Collection => ({
|
|
id,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id]),
|
|
name,
|
|
type: "album",
|
|
updationTime: 1,
|
|
isShared,
|
|
});
|
|
|
|
const file = (id: number, collectionID: number, title: string): EnteFile => ({
|
|
id,
|
|
collectionID,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
metadata: {
|
|
title,
|
|
fileType: "image",
|
|
creationTime: 1000,
|
|
modificationTime: 1000,
|
|
},
|
|
file: { decryptionHeader: "aGVhZGVy" },
|
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
|
updationTime: 1,
|
|
});
|
|
|
|
const COLLECTIONS = [collection(1, "Vacation"), collection(2, "Work", true)];
|
|
|
|
const FILES: Record<number, EnteFile[]> = {
|
|
1: [file(100, 1, "beach.jpg"), file(101, 1, "sunset.jpg")],
|
|
2: [file(200, 2, "diagram.png")],
|
|
};
|
|
|
|
// An original is 7 bytes and a thumbnail 3. `failID` makes that file's
|
|
// original fail; `emptyThumbID` makes the server report that file's
|
|
// thumbnail as empty. `withNewFile` adds new.jpg (102) to Vacation, advancing
|
|
// the collection's updationTime as the server does. `beachInWork` adds
|
|
// beach.jpg (100) to Work as well, so that one file is in both albums.
|
|
// `refreshError` makes listing collections fail with that message.
|
|
const fakeClient = (
|
|
opts: {
|
|
failID?: number;
|
|
emptyThumbID?: number;
|
|
withNewFile?: boolean;
|
|
beachInWork?: boolean;
|
|
refreshError?: string;
|
|
} = {},
|
|
) => {
|
|
const collections = opts.withNewFile
|
|
? [{ ...COLLECTIONS[0], updationTime: 2 }, COLLECTIONS[1]]
|
|
: COLLECTIONS;
|
|
let files = FILES;
|
|
if (opts.withNewFile) {
|
|
files = {
|
|
...files,
|
|
1: [...files[1], { ...file(102, 1, "new.jpg"), updationTime: 2 }],
|
|
};
|
|
}
|
|
if (opts.beachInWork) {
|
|
files = { ...files, 2: [...files[2], file(100, 2, "beach.jpg")] };
|
|
}
|
|
const source: ContentSource = {
|
|
original: async ({ file: f, destination }) => {
|
|
if (f.id === opts.failID) throw new Error("HTTP 500 from server");
|
|
writeFileSync(destination, Buffer.alloc(7, f.id & 0xff));
|
|
return { bytesWritten: 7 };
|
|
},
|
|
thumbnail: async ({ file: f, destination }) => {
|
|
writeFileSync(destination, Buffer.alloc(3, f.id & 0xff));
|
|
return { bytesWritten: 3 };
|
|
},
|
|
};
|
|
const fake = {
|
|
whoami: () => ({ email: "cli@example.com", userID: USER_ID }),
|
|
collectionsSince: async () => {
|
|
if (opts.refreshError) throw new Error(opts.refreshError);
|
|
return { collections, deleted: [], cursor: 1 };
|
|
},
|
|
filesSince: async (args: { collectionID: number }) => ({
|
|
files: files[args.collectionID] ?? [],
|
|
deleted: [],
|
|
cursor: 1,
|
|
}),
|
|
contentSource: () => source,
|
|
getApiClient: () => ({
|
|
// The ML data request of `backup-metadata`: no file has any.
|
|
postJSON: async () => ({ data: [] }),
|
|
getThumbnailStream: async (fileID: number) =>
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
if (fileID !== opts.emptyThumbID) {
|
|
controller.enqueue(new Uint8Array(3));
|
|
}
|
|
controller.close();
|
|
},
|
|
}),
|
|
}),
|
|
};
|
|
// The commands only call the methods above.
|
|
return fake as unknown as Client;
|
|
};
|
|
|
|
// What a read command prints on stderr while the refresh it waits for fetches
|
|
// the files of each of the fake's two albums, on an empty cache.
|
|
const FETCHED_BOTH_ALBUMS =
|
|
"Fetched the files of 1 of 2 collections\n" +
|
|
"Fetched the files of 2 of 2 collections\n";
|
|
|
|
// The same for an account with one album.
|
|
const FETCHED_ONE_ALBUM = "Fetched the files of 1 of 1 collections\n";
|
|
|
|
// Collects everything written to it.
|
|
class Output {
|
|
text = "";
|
|
write(text: string): void {
|
|
this.text += text;
|
|
}
|
|
}
|
|
|
|
let root: string;
|
|
let stdout: Output;
|
|
let stderr: Output;
|
|
|
|
// The commands log at the default level, info, as `quak` without `-v` or `-q`
|
|
// does; the levels have their own tests, in `log.test.ts`.
|
|
const context = (client: Client | null = fakeClient()): CliContext => ({
|
|
stdout,
|
|
log: stderrLog(stderr, "info"),
|
|
sessionDir: join(root, "session"),
|
|
cacheDir: join(root, "cache"),
|
|
loadSession: () => client,
|
|
login: async () => {
|
|
throw new Error("login not expected");
|
|
},
|
|
prompt: async () => {
|
|
throw new Error("prompt not expected");
|
|
},
|
|
promptSecret: async () => {
|
|
throw new Error("prompt not expected");
|
|
},
|
|
});
|
|
|
|
beforeAll(async () => {
|
|
await init();
|
|
});
|
|
|
|
beforeEach(() => {
|
|
root = mkdtempSync(join(tmpdir(), "quak-cli-test-"));
|
|
stdout = new Output();
|
|
stderr = new Output();
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(root, { recursive: true, force: true });
|
|
});
|
|
|
|
describe("session file", () => {
|
|
const snapshot: ClientSnapshot = {
|
|
email: "cli@example.com",
|
|
userID: USER_ID,
|
|
token: "token",
|
|
masterKey: "a",
|
|
secretKey: "b",
|
|
publicKey: "c",
|
|
};
|
|
|
|
it("is written with mode 0600 in a directory with mode 0700", () => {
|
|
const dir = join(root, "new", "session");
|
|
saveSession(dir, snapshot);
|
|
expect(statSync(dir).mode & 0o777).toBe(0o700);
|
|
const path = join(dir, "session.json");
|
|
expect(statSync(path).mode & 0o777).toBe(0o600);
|
|
expect(JSON.parse(readFileSync(path, "utf-8"))).toEqual(snapshot);
|
|
});
|
|
|
|
it("a missing session exits 3 with 'Not logged in' from every command that needs one", async () => {
|
|
const ctx = { ...context(), loadSession };
|
|
const dir = join(root, "backup");
|
|
expect(await whoamiCommand(ctx)).toBe(3);
|
|
expect(await collectionsCommand(ctx, {})).toBe(3);
|
|
expect(await filesCommand(ctx, "1", {})).toBe(3);
|
|
expect(await getCommand(ctx, "100", {})).toBe(3);
|
|
expect(await getThumbCommand(ctx, "100", {})).toBe(3);
|
|
expect(await backupMetadataCommand(ctx, dir, {})).toBe(3);
|
|
expect(await backupCommand(ctx, dir, {})).toBe(3);
|
|
expect(await listMissingThumbnailsCommand(ctx, {})).toBe(3);
|
|
expect(await fixMissingThumbnailsCommand(ctx, {})).toBe(3);
|
|
const notLoggedIn =
|
|
`Not logged in. Run "quak login" first.\n` +
|
|
`Session file: ${join(ctx.sessionDir, "session.json")}\n`;
|
|
expect(stderr.text).toBe(notLoggedIn.repeat(9));
|
|
expect(stdout.text).toBe("");
|
|
expect(existsSync(dir)).toBe(false);
|
|
});
|
|
|
|
it("a corrupt session exits 3 and says it is corrupt", async () => {
|
|
const ctx = { ...context(), loadSession };
|
|
saveSession(ctx.sessionDir, snapshot);
|
|
expect(await collectionsCommand(ctx, {})).toBe(3);
|
|
expect(stderr.text).toContain("is corrupt");
|
|
expect(stderr.text).toContain(
|
|
`Run "quak logout" and then "quak login" to replace it.\n`,
|
|
);
|
|
expect(stdout.text).toBe("");
|
|
});
|
|
});
|
|
|
|
// The login function is a fake that hands back a client whose snapshot is
|
|
// `snapshot`; each prompt is recorded and answered with "123456".
|
|
describe("login", () => {
|
|
const snapshot: ClientSnapshot = {
|
|
email: "cli@example.com",
|
|
userID: USER_ID,
|
|
token: "token",
|
|
masterKey: "a",
|
|
secretKey: "b",
|
|
publicKey: "c",
|
|
};
|
|
|
|
const loggedIn = {
|
|
whoami: () => ({ email: "cli@example.com", userID: USER_ID }),
|
|
toJSON: () => snapshot,
|
|
} as unknown as Client;
|
|
|
|
let prompts: string[];
|
|
|
|
const loginContext = (
|
|
login: (opts: LoginOptions) => Promise<Client>,
|
|
): CliContext => ({
|
|
...context(),
|
|
login,
|
|
prompt: async (message) => {
|
|
prompts.push(message);
|
|
return "123456";
|
|
},
|
|
promptSecret: async (message) => {
|
|
prompts.push(message);
|
|
return "123456";
|
|
},
|
|
});
|
|
|
|
beforeEach(() => {
|
|
prompts = [];
|
|
vi.stubEnv("QUAK_EMAIL", "cli@example.com");
|
|
vi.stubEnv("QUAK_PASSWORD", "hunter2");
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.unstubAllEnvs();
|
|
});
|
|
|
|
it("takes the email and password from the environment without a prompt", async () => {
|
|
const calls: LoginOptions[] = [];
|
|
const ctx = loginContext(async (opts) => {
|
|
calls.push(opts);
|
|
return loggedIn;
|
|
});
|
|
expect(await loginCommand(ctx)).toBe(0);
|
|
|
|
expect(prompts).toEqual([]);
|
|
expect(calls).toHaveLength(1);
|
|
expect(calls[0]!.email).toBe("cli@example.com");
|
|
expect(calls[0]!.password).toBe("hunter2");
|
|
const path = join(ctx.sessionDir, "session.json");
|
|
expect(stderr.text).toBe(
|
|
"Authenticating...\n" +
|
|
`Logged in as cli@example.com (user ${USER_ID})\n` +
|
|
`Session saved to ${path}\n`,
|
|
);
|
|
});
|
|
|
|
it("saves the session with mode 0600 in a directory with mode 0700", async () => {
|
|
const ctx = loginContext(async () => loggedIn);
|
|
expect(await loginCommand(ctx)).toBe(0);
|
|
|
|
expect(statSync(ctx.sessionDir).mode & 0o777).toBe(0o700);
|
|
const path = join(ctx.sessionDir, "session.json");
|
|
expect(statSync(path).mode & 0o777).toBe(0o600);
|
|
expect(JSON.parse(readFileSync(path, "utf-8"))).toEqual(snapshot);
|
|
});
|
|
|
|
it("asks for the TOTP code when the account needs one", async () => {
|
|
let code: string | undefined;
|
|
const ctx = loginContext(async (opts) => {
|
|
code = await opts.totp!();
|
|
return loggedIn;
|
|
});
|
|
expect(await loginCommand(ctx)).toBe(0);
|
|
|
|
expect(prompts).toEqual(["TOTP code: "]);
|
|
expect(code).toBe("123456");
|
|
});
|
|
|
|
it("a failed login exits 1, says why and writes no session", async () => {
|
|
const ctx = loginContext(async () => {
|
|
throw new Error("HTTP 401 from server");
|
|
});
|
|
expect(await loginCommand(ctx)).toBe(1);
|
|
|
|
expect(stderr.text).toBe(
|
|
"Authenticating...\nLogin failed: HTTP 401 from server\n",
|
|
);
|
|
expect(existsSync(join(ctx.sessionDir, "session.json"))).toBe(false);
|
|
});
|
|
});
|
|
|
|
// These use a real client read from the session file, over a fake API that
|
|
// records each request and answers with `status`.
|
|
describe("logout", () => {
|
|
const snapshot: ClientSnapshot = {
|
|
email: "cli@example.com",
|
|
userID: USER_ID,
|
|
token: "saved-token",
|
|
masterKey: toBase64(new Uint8Array(32)),
|
|
secretKey: toBase64(new Uint8Array(32)),
|
|
publicKey: toBase64(new Uint8Array(32)),
|
|
};
|
|
|
|
const requests: Request[] = [];
|
|
|
|
const logoutContext = (status: number): CliContext => ({
|
|
...context(),
|
|
loadSession: (path) =>
|
|
loadSession(path, {
|
|
fetch: async (url, init) => {
|
|
requests.push(new Request(url, init));
|
|
return new Response(JSON.stringify({}), {
|
|
status,
|
|
headers: { "content-type": "application/json" },
|
|
});
|
|
},
|
|
}),
|
|
});
|
|
|
|
beforeEach(() => {
|
|
requests.length = 0;
|
|
});
|
|
|
|
it("ends the session on the server, then deletes the file", async () => {
|
|
const ctx = logoutContext(200);
|
|
saveSession(ctx.sessionDir, snapshot);
|
|
expect(await logoutCommand(ctx)).toBe(0);
|
|
|
|
expect(requests).toHaveLength(1);
|
|
expect(requests[0]!.method).toBe("POST");
|
|
expect(new URL(requests[0]!.url).pathname).toBe("/users/logout");
|
|
expect(requests[0]!.headers.get("X-Auth-Token")).toBe("saved-token");
|
|
expect(existsSync(join(ctx.sessionDir, "session.json"))).toBe(false);
|
|
expect(stderr.text).toBe(
|
|
"Session ended on the server.\n" +
|
|
"Session deleted.\n" +
|
|
`Cache directory ${ctx.cacheDir} still holds decrypted data; delete it to remove that data.\n`,
|
|
);
|
|
});
|
|
|
|
it("still deletes the file when the server call fails, and says so", async () => {
|
|
const ctx = logoutContext(500);
|
|
saveSession(ctx.sessionDir, snapshot);
|
|
expect(await logoutCommand(ctx)).toBe(1);
|
|
|
|
expect(requests).toHaveLength(1);
|
|
expect(existsSync(join(ctx.sessionDir, "session.json"))).toBe(false);
|
|
expect(stderr.text).toBe(
|
|
"Could not end the session on the server: HTTP 500\n" +
|
|
"Session deleted.\n" +
|
|
`Cache directory ${ctx.cacheDir} still holds decrypted data; delete it to remove that data.\n`,
|
|
);
|
|
});
|
|
|
|
it("names the account's default cache directory without --cache-dir", async () => {
|
|
const ctx = { ...logoutContext(200), cacheDir: undefined };
|
|
saveSession(ctx.sessionDir, snapshot);
|
|
expect(await logoutCommand(ctx)).toBe(0);
|
|
expect(stderr.text).toContain(
|
|
`Cache directory ${defaultCacheDirectory(USER_ID)} still holds decrypted data`,
|
|
);
|
|
});
|
|
|
|
it("without a session says so, calls nothing and exits 0", async () => {
|
|
expect(await logoutCommand(logoutContext(200))).toBe(0);
|
|
expect(requests).toHaveLength(0);
|
|
expect(stderr.text).toBe("No session found.\n");
|
|
});
|
|
});
|
|
|
|
describe("whoami", () => {
|
|
it("prints the account as one line of JSON", async () => {
|
|
expect(await whoamiCommand(context())).toBe(0);
|
|
expect(stdout.text).toBe(
|
|
`{"email":"cli@example.com","userID":${USER_ID}}\n`,
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("collections", () => {
|
|
it("prints one tab-separated line per album", async () => {
|
|
expect(await collectionsCommand(context(), {})).toBe(0);
|
|
expect(stdout.text).toBe(
|
|
"1\talbum\tVacation\n" + "2\talbum\tWork (shared)\n",
|
|
);
|
|
});
|
|
|
|
it("prints a JSON array with --json", async () => {
|
|
expect(await collectionsCommand(context(), { json: true })).toBe(0);
|
|
expect(JSON.parse(stdout.text)).toEqual([
|
|
{
|
|
id: 1,
|
|
name: "Vacation",
|
|
type: "album",
|
|
ownerID: USER_ID,
|
|
isShared: false,
|
|
updationTime: 1,
|
|
},
|
|
{
|
|
id: 2,
|
|
name: "Work",
|
|
type: "album",
|
|
ownerID: USER_ID,
|
|
isShared: true,
|
|
updationTime: 1,
|
|
},
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe("files", () => {
|
|
it("prints one tab-separated line per file", async () => {
|
|
expect(await filesCommand(context(), "1", {})).toBe(0);
|
|
expect(stdout.text).toBe(
|
|
"100\timage\tbeach.jpg\n" + "101\timage\tsunset.jpg\n",
|
|
);
|
|
});
|
|
|
|
it("prints a JSON array with --json", async () => {
|
|
const code = await filesCommand(context(), "2", { json: true });
|
|
expect(code).toBe(0);
|
|
expect(JSON.parse(stdout.text)).toEqual([
|
|
{
|
|
id: 200,
|
|
title: "diagram.png",
|
|
fileType: "image",
|
|
creationTime: 1000,
|
|
collectionID: 2,
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("exits 1 for an unknown collection", async () => {
|
|
expect(await filesCommand(context(), "9", {})).toBe(1);
|
|
expect(stderr.text).toBe("Collection 9 not found\n");
|
|
});
|
|
|
|
// Without a collection ID it lists every file in the account once (issue
|
|
// #211). beach.jpg is in Vacation and Work, and Vacation comes first, so it
|
|
// is listed once, as Vacation's.
|
|
it("without a collection ID prints every file once", async () => {
|
|
const ctx = context(fakeClient({ beachInWork: true }));
|
|
expect(await filesCommand(ctx, undefined, {})).toBe(0);
|
|
expect(stdout.text).toBe(
|
|
"100\timage\tbeach.jpg\n" +
|
|
"101\timage\tsunset.jpg\n" +
|
|
"200\timage\tdiagram.png\n",
|
|
);
|
|
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS);
|
|
});
|
|
|
|
it("without a collection ID gives each file the first album holding it with --json", async () => {
|
|
const ctx = context(fakeClient({ beachInWork: true }));
|
|
expect(await filesCommand(ctx, undefined, { json: true })).toBe(0);
|
|
expect(
|
|
JSON.parse(stdout.text).map(
|
|
(row: { id: number; collectionID: number }) => [
|
|
row.id,
|
|
row.collectionID,
|
|
],
|
|
),
|
|
).toEqual([
|
|
[100, 1],
|
|
[101, 1],
|
|
[200, 2],
|
|
]);
|
|
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS);
|
|
});
|
|
});
|
|
|
|
describe("get and get-thumb", () => {
|
|
it("get finds a file in any album without --collection", async () => {
|
|
const out = join(root, "diagram.png");
|
|
expect(await getCommand(context(), "200", { out })).toBe(0);
|
|
expect(readFileSync(out)).toEqual(Buffer.alloc(7, 200));
|
|
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS + `7 bytes -> ${out}\n`);
|
|
});
|
|
|
|
it("get-thumb finds a file in any album without --collection", async () => {
|
|
const out = join(root, "thumb.jpg");
|
|
expect(await getThumbCommand(context(), "200", { out })).toBe(0);
|
|
expect(readFileSync(out)).toEqual(Buffer.alloc(3, 200));
|
|
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS + `3 bytes -> ${out}\n`);
|
|
});
|
|
|
|
it("get exits 1 when no album has the file", async () => {
|
|
const out = join(root, "x");
|
|
expect(await getCommand(context(), "999", { out })).toBe(1);
|
|
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS + "File 999 not found\n");
|
|
expect(existsSync(out)).toBe(false);
|
|
});
|
|
|
|
it("get-thumb exits 1 when no album has the file", async () => {
|
|
const out = join(root, "x");
|
|
expect(await getThumbCommand(context(), "999", { out })).toBe(1);
|
|
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS + "File 999 not found\n");
|
|
expect(existsSync(out)).toBe(false);
|
|
});
|
|
});
|
|
|
|
// A file system refuses a file name over 255 bytes. The account here is one
|
|
// album, whose name is 300 bytes, holding one file, 700, with the title given.
|
|
describe("names from an album name or a title over 255 bytes", () => {
|
|
const longNames = (title: string): Client =>
|
|
({
|
|
...fakeClient(),
|
|
collectionsSince: async () => ({
|
|
collections: [collection(7, "A".repeat(300))],
|
|
deleted: [],
|
|
cursor: 1,
|
|
}),
|
|
filesSince: async () => ({
|
|
files: [file(700, 7, title)],
|
|
deleted: [],
|
|
cursor: 1,
|
|
}),
|
|
}) as unknown as Client;
|
|
const longTitle = `${"t".repeat(296)}.jpg`;
|
|
|
|
// Run `command` in a new directory, where `get` and `get-thumb` without
|
|
// `--out` write, and return its exit code and the directory.
|
|
const inNewDirectory = async (
|
|
command: () => Promise<number>,
|
|
): Promise<{ code: number; dir: string }> => {
|
|
const dir = mkdtempSync(join(root, "cwd-"));
|
|
const previous = process.cwd();
|
|
process.chdir(dir);
|
|
try {
|
|
return { code: await command(), dir };
|
|
} finally {
|
|
process.chdir(previous);
|
|
}
|
|
};
|
|
|
|
it("get cuts the title short to 255 bytes, keeping its extension, and exits 0", async () => {
|
|
const { code, dir } = await inNewDirectory(() =>
|
|
getCommand(context(longNames(longTitle)), "700", {}),
|
|
);
|
|
|
|
const name = `${"t".repeat(251)}.jpg`;
|
|
expect(Buffer.byteLength(name)).toBe(255);
|
|
expect(code).toBe(0);
|
|
expect(readdirSync(dir)).toEqual([name]);
|
|
expect(stderr.text).toBe(FETCHED_ONE_ALBUM + `7 bytes -> ${name}\n`);
|
|
});
|
|
|
|
it("get-thumb cuts the title short so that thumb_ and it fit in 255 bytes, and exits 0", async () => {
|
|
const { code, dir } = await inNewDirectory(() =>
|
|
getThumbCommand(context(longNames(longTitle)), "700", {}),
|
|
);
|
|
|
|
const name = `thumb_${"t".repeat(245)}.jpg`;
|
|
expect(Buffer.byteLength(name)).toBe(255);
|
|
expect(code).toBe(0);
|
|
expect(readdirSync(dir)).toEqual([name]);
|
|
expect(stderr.text).toBe(FETCHED_ONE_ALBUM + `3 bytes -> ${name}\n`);
|
|
});
|
|
|
|
it("backup-metadata cuts the album name short so that its folder's name fits in 255 bytes, and exits 0", async () => {
|
|
const dir = join(root, "dump");
|
|
|
|
const code = await backupMetadataCommand(
|
|
context(longNames("beach.jpg")),
|
|
dir,
|
|
{},
|
|
);
|
|
|
|
const folder = `7-${"A".repeat(253)}`;
|
|
expect(Buffer.byteLength(folder)).toBe(255);
|
|
expect(code).toBe(0);
|
|
expect(readdirSync(join(dir, "collections"))).toEqual([folder]);
|
|
expect(readdirSync(join(dir, "collections", folder)).sort()).toEqual([
|
|
"700.json",
|
|
"_collection.json",
|
|
]);
|
|
});
|
|
|
|
// The title's extension is 300 letters, too long for the save path's
|
|
// name, so the original is saved with `.bin`.
|
|
it("backup saves a title with a 300-letter extension as .bin, and exits 0", async () => {
|
|
const dir = join(root, "backup");
|
|
const title = `b.${"e".repeat(300)}`;
|
|
|
|
const code = await backupCommand(context(longNames(title)), dir, {});
|
|
|
|
const path = savePath(dir, file(700, 7, title));
|
|
expect(code).toBe(0);
|
|
expect(basename(path)).toMatch(/^\d{4}-\d{2}-\d{2}\.700\.bin$/);
|
|
expect(readFileSync(path)).toEqual(Buffer.alloc(7, 700 & 0xff));
|
|
});
|
|
});
|
|
|
|
// A live photo, which Ente stores as one ZIP, is written as its image and its
|
|
// video, which a photo viewer can open. The account here is Vacation holding
|
|
// one live photo, 300, downloaded through the real download layer
|
|
// (test/live-photo.ts).
|
|
describe("a live photo", () => {
|
|
const livePhotoClient = async (
|
|
image = IMAGE,
|
|
title = "IMG_0300.HEIC",
|
|
): Promise<Client> => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(300, 1, title),
|
|
livePhotoZip({ "image.heic": image, "video.mov": VIDEO }),
|
|
livePhotoHash(image, VIDEO),
|
|
);
|
|
const fake = {
|
|
whoami: () => ({ email: "cli@example.com", userID: USER_ID }),
|
|
collectionsSince: async () => ({
|
|
collections: [collection(1, "Vacation")],
|
|
deleted: [],
|
|
cursor: 1,
|
|
}),
|
|
filesSince: async () => ({ files: [live], deleted: [], cursor: 1 }),
|
|
contentSource: () => cdnSource(new Map([[300, body]])),
|
|
// The ML data request of `backup-metadata`: no file has any.
|
|
getApiClient: () => ({ postJSON: async () => ({ data: [] }) }),
|
|
};
|
|
return fake as unknown as Client;
|
|
};
|
|
|
|
it("get writes its image and video, named after the title with their own extensions", async () => {
|
|
const ctx = context(await livePhotoClient());
|
|
const dir = join(root, "cwd");
|
|
mkdirSync(dir);
|
|
const previous = process.cwd();
|
|
process.chdir(dir);
|
|
try {
|
|
expect(await getCommand(ctx, "300", {})).toBe(0);
|
|
} finally {
|
|
process.chdir(previous);
|
|
}
|
|
|
|
expect(readdirSync(dir).sort()).toEqual([
|
|
"IMG_0300.heic",
|
|
"IMG_0300.mov",
|
|
]);
|
|
expect(readFileSync(join(dir, "IMG_0300.heic"))).toEqual(
|
|
Buffer.from(IMAGE),
|
|
);
|
|
expect(readFileSync(join(dir, "IMG_0300.mov"))).toEqual(
|
|
Buffer.from(VIDEO),
|
|
);
|
|
expect(stderr.text).toBe(
|
|
FETCHED_ONE_ALBUM +
|
|
`${IMAGE.length} bytes -> IMG_0300.heic\n` +
|
|
`${VIDEO.length} bytes -> IMG_0300.mov\n`,
|
|
);
|
|
});
|
|
|
|
// Run `get` without `--out` in a new directory for a live photo titled
|
|
// `title`, and return its exit code and the names it wrote.
|
|
const getInNewDirectory = async (
|
|
title: string,
|
|
): Promise<{ code: number; names: string[] }> => {
|
|
const ctx = context(await livePhotoClient(IMAGE, title));
|
|
const dir = mkdtempSync(join(root, "cwd-"));
|
|
const previous = process.cwd();
|
|
process.chdir(dir);
|
|
try {
|
|
const code = await getCommand(ctx, "300", {});
|
|
return { code, names: readdirSync(dir).sort() };
|
|
} finally {
|
|
process.chdir(previous);
|
|
}
|
|
};
|
|
const imageName = `${"t".repeat(250)}.heic`;
|
|
const videoName = `${"t".repeat(251)}.mov`;
|
|
|
|
// The title fits, but ".heic" is one byte longer than ".jpg", so the
|
|
// image's name is cut short once its extension is in it.
|
|
it("get cuts its image's name short for a 255-byte title ending in .jpg, and exits 0", async () => {
|
|
const title = `${"t".repeat(251)}.jpg`;
|
|
|
|
const result = await getInNewDirectory(title);
|
|
|
|
expect(Buffer.byteLength(title)).toBe(255);
|
|
expect(Buffer.byteLength(imageName)).toBe(255);
|
|
expect(Buffer.byteLength(videoName)).toBe(255);
|
|
expect(result).toEqual({ code: 0, names: [imageName, videoName] });
|
|
});
|
|
|
|
it("get cuts the names of its image and video short for a 300-byte title with no extension, and exits 0", async () => {
|
|
const result = await getInNewDirectory("t".repeat(300));
|
|
|
|
expect(result).toEqual({ code: 0, names: [imageName, videoName] });
|
|
});
|
|
|
|
it("get --out writes the image there and the video beside it", async () => {
|
|
const out = join(root, "photo.jpg");
|
|
const video = join(root, "photo.mov");
|
|
|
|
expect(
|
|
await getCommand(context(await livePhotoClient()), "300", { out }),
|
|
).toBe(0);
|
|
|
|
expect(readFileSync(out)).toEqual(Buffer.from(IMAGE));
|
|
expect(readFileSync(video)).toEqual(Buffer.from(VIDEO));
|
|
expect(stderr.text).toBe(
|
|
FETCHED_ONE_ALBUM +
|
|
`${IMAGE.length} bytes -> ${out}\n${VIDEO.length} bytes -> ${video}\n`,
|
|
);
|
|
});
|
|
|
|
it("get exits 1 and writes nothing when --out has the video's extension", async () => {
|
|
const out = join(root, "photo.MOV");
|
|
|
|
expect(
|
|
await getCommand(context(await livePhotoClient()), "300", { out }),
|
|
).toBe(1);
|
|
|
|
expect(existsSync(out)).toBe(false);
|
|
expect(existsSync(join(root, "photo.mov"))).toBe(false);
|
|
expect(stderr.text).toBe(
|
|
FETCHED_ONE_ALBUM +
|
|
`File 300 is a live photo, and its video would also be written to ${out}\n`,
|
|
);
|
|
});
|
|
|
|
it("backup-metadata --exif reads its image", async () => {
|
|
// A 4x4 JPEG, whose size can only come from reading the image; the
|
|
// ZIP and the video are not JPEGs.
|
|
const jpeg = jpegJs.encode(
|
|
{ data: new Uint8Array(4 * 4 * 4), width: 4, height: 4 },
|
|
50,
|
|
).data;
|
|
const dir = join(root, "dump");
|
|
|
|
expect(
|
|
await backupMetadataCommand(
|
|
context(await livePhotoClient(new Uint8Array(jpeg))),
|
|
dir,
|
|
{ exif: true },
|
|
),
|
|
).toBe(0);
|
|
|
|
const record = JSON.parse(
|
|
readFileSync(
|
|
join(dir, "collections", "1-Vacation", "300.json"),
|
|
"utf-8",
|
|
),
|
|
);
|
|
expect(record.imageMetadata).toMatchObject({
|
|
format: "jpeg",
|
|
width: 4,
|
|
height: 4,
|
|
});
|
|
});
|
|
|
|
it("backup-metadata --exif records the EXIF of a HEIC image", async () => {
|
|
const dir = join(root, "dump");
|
|
|
|
expect(
|
|
await backupMetadataCommand(
|
|
context(await livePhotoClient(HEIC_WITH_EXIF)),
|
|
dir,
|
|
{ exif: true },
|
|
),
|
|
).toBe(0);
|
|
|
|
const record = JSON.parse(
|
|
readFileSync(
|
|
join(dir, "collections", "1-Vacation", "300.json"),
|
|
"utf-8",
|
|
),
|
|
);
|
|
expect(record.imageMetadata.exifError).toBeUndefined();
|
|
expect(record.imageMetadata.exif).toMatchObject({
|
|
Make: { value: ["Canon"] },
|
|
Model: { value: ["EOS R5"] },
|
|
DateTimeOriginal: { value: ["2021:07:15 14:30:00"] },
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("backup", () => {
|
|
it("exits 0 and prints a summary when every file is saved", async () => {
|
|
const dir = join(root, "backup");
|
|
expect(await backupCommand(context(), dir, {})).toBe(0);
|
|
expect(stderr.text).toContain(
|
|
"\n--- Backup complete ---\n" +
|
|
" Total files: 3\n" +
|
|
" Downloaded: 3\n" +
|
|
" Skipped: 0\n" +
|
|
" Failed: 0\n",
|
|
);
|
|
expect(stdout.text).toBe("");
|
|
expect(existsSync(join(dir, "backup.lock"))).toBe(false);
|
|
});
|
|
|
|
// The backup opens its library with the precache off: it fetches the
|
|
// originals it needs into the backup, and must not also fetch every
|
|
// thumbnail in the account, or keep originals, in the per-user cache.
|
|
it("leaves nothing in the cache's originals and thumbnails", async () => {
|
|
const dir = join(root, "backup");
|
|
expect(await backupCommand(context(), dir, {})).toBe(0);
|
|
expect(readdirSync(join(root, "cache", "thumbnails"))).toEqual([]);
|
|
expect(readdirSync(join(root, "cache", "originals"))).toEqual([]);
|
|
});
|
|
|
|
it("exits 1 and lists the file when one download fails", async () => {
|
|
const ctx = context(fakeClient({ failID: 101 }));
|
|
expect(await backupCommand(ctx, join(root, "backup"), {})).toBe(1);
|
|
expect(stderr.text).toContain(" Failed: 1\n");
|
|
expect(stderr.text).toContain(
|
|
"\nFailed files:\n" +
|
|
" [Vacation] sunset.jpg (id 101): HTTP 500 from server\n",
|
|
);
|
|
});
|
|
|
|
// A failed file is an error line both where the backup fails it and in
|
|
// the list at the end. The last line is fatal, so `-q`, which prints
|
|
// fatal lines only, still says why the backup exits 1.
|
|
it("logs a failed file at error, and ends with one fatal line counting the failures", async () => {
|
|
const dir = join(root, "backup");
|
|
const last = `quak: 1 file(s) and 0 album(s) failed; see ${join(dir, "failures.json")} and the FAILED lines\n`;
|
|
const ctx = context(fakeClient({ failID: 101 }));
|
|
expect(
|
|
await backupCommand(
|
|
{ ...ctx, log: stderrLog(stderr, "error") },
|
|
dir,
|
|
{},
|
|
),
|
|
).toBe(1);
|
|
expect(stderr.text).toBe(
|
|
"FAILED original sunset.jpg: HTTP 500 from server\n" +
|
|
"\nFailed files:\n" +
|
|
" [Vacation] sunset.jpg (id 101): HTTP 500 from server\n" +
|
|
last,
|
|
);
|
|
|
|
stderr.text = "";
|
|
expect(
|
|
await backupCommand(
|
|
{ ...ctx, log: stderrLog(stderr, "fatal") },
|
|
dir,
|
|
{ json: true },
|
|
),
|
|
).toBe(1);
|
|
expect(stderr.text).toBe(last);
|
|
});
|
|
|
|
// `--json` shapes stdout only: the log lines stay on stderr, and the JSON
|
|
// takes the place of the summary.
|
|
it("prints the result as JSON with --json, still exiting 1 on a failure", async () => {
|
|
const ctx = context(fakeClient({ failID: 101 }));
|
|
const code = await backupCommand(ctx, join(root, "backup"), {
|
|
json: true,
|
|
});
|
|
expect(code).toBe(1);
|
|
const result = JSON.parse(stdout.text);
|
|
expect(result).toMatchObject({
|
|
totalFiles: 3,
|
|
downloaded: 2,
|
|
skipped: 0,
|
|
failed: 1,
|
|
});
|
|
expect(result.errors[0].fileID).toBe(101);
|
|
expect(stderr.text).toContain(
|
|
"Fetching original sunset.jpg (101)...\n",
|
|
);
|
|
expect(stderr.text).not.toContain("Backup complete");
|
|
});
|
|
|
|
// On an empty cache the library's first refresh fetches the files of
|
|
// every album before the backup can start, which on a large account takes
|
|
// a while; each album it has fetched the files of is counted on stderr,
|
|
// with `--json` too.
|
|
it("counts on stderr each album whose files its first refresh fetched, with --json too", async () => {
|
|
const start =
|
|
"Starting backup...\n" +
|
|
FETCHED_BOTH_ALBUMS +
|
|
"Refreshing library...\n";
|
|
for (const json of [false, true]) {
|
|
stderr.text = "";
|
|
const dir = mkdtempSync(join(root, "backup-"));
|
|
const ctx = {
|
|
...context(),
|
|
cacheDir: mkdtempSync(join(root, "cache-")),
|
|
};
|
|
expect(await backupCommand(ctx, dir, { json })).toBe(0);
|
|
expect(stderr.text.slice(0, start.length)).toBe(start);
|
|
}
|
|
});
|
|
|
|
it("--verify downloads again an original that does not match its hash, prints the counts, and exits 0", async () => {
|
|
// Each file records the hash of the original the fake writes for it.
|
|
const client = {
|
|
...fakeClient(),
|
|
filesSince: async (args: { collectionID: number }) => ({
|
|
files: (FILES[args.collectionID] ?? []).map((f) => ({
|
|
...f,
|
|
metadata: {
|
|
...f.metadata,
|
|
hash: blake2b(Buffer.alloc(7, f.id & 0xff)),
|
|
},
|
|
})),
|
|
deleted: [],
|
|
cursor: 1,
|
|
}),
|
|
} as unknown as Client;
|
|
const ctx = context(client);
|
|
const dir = join(root, "backup");
|
|
expect(await backupCommand(ctx, dir, {})).toBe(0);
|
|
writeFileSync(savePath(dir, FILES[1]![0]!), "corrupt");
|
|
|
|
expect(await backupCommand(ctx, dir, { verify: true })).toBe(0);
|
|
|
|
expect(stderr.text).toContain(
|
|
"MISMATCH original beach.jpg (100): its bytes do not match its content hash\n",
|
|
);
|
|
expect(stderr.text).toContain(
|
|
" Downloaded: 1\n" +
|
|
" Skipped: 2\n" +
|
|
" Verified: 2\n" +
|
|
" Mismatched: 1\n" +
|
|
" Unchecked: 0\n" +
|
|
" Failed: 0\n",
|
|
);
|
|
});
|
|
|
|
it("--verify --json adds the verified, mismatched and unchecked counts", async () => {
|
|
const dir = join(root, "backup");
|
|
expect(await backupCommand(context(), dir, {})).toBe(0);
|
|
|
|
const code = await backupCommand(context(), dir, {
|
|
verify: true,
|
|
json: true,
|
|
});
|
|
|
|
expect(code).toBe(0);
|
|
expect(JSON.parse(stdout.text)).toMatchObject({
|
|
skipped: 3,
|
|
verified: 0,
|
|
mismatched: 0,
|
|
unchecked: 3,
|
|
failed: 0,
|
|
});
|
|
});
|
|
|
|
it("exits 1 and lists each file when the ML data fetch fails", async () => {
|
|
const client = {
|
|
...fakeClient(),
|
|
fetchMLData: async () => {
|
|
throw new Error("HTTP 503 from server");
|
|
},
|
|
} as unknown as Client;
|
|
expect(
|
|
await backupCommand(context(client), join(root, "backup"), {}),
|
|
).toBe(1);
|
|
expect(stderr.text).toContain(" Failed: 3\n");
|
|
expect(stderr.text).toContain(
|
|
" [Vacation] beach.jpg (id 100): ML data: HTTP 503 from server\n",
|
|
);
|
|
});
|
|
|
|
// Runs `backup` through `run`, as `bin/quak.ts` does, which prints a thrown
|
|
// error; returns the exit code and what `run` printed.
|
|
const backupThroughRun = async (
|
|
ctx: CliContext,
|
|
dir: string,
|
|
): Promise<{ code: number; runText: string }> => {
|
|
const runStderr = new PassThrough();
|
|
let runText = "";
|
|
runStderr.on("data", (chunk: Buffer) => {
|
|
runText += chunk.toString();
|
|
});
|
|
const code = await new Promise<number>((resolve) => {
|
|
void run(
|
|
backupCommand(ctx, dir, {}),
|
|
new PassThrough(),
|
|
runStderr,
|
|
stderrLog(runStderr, "info"),
|
|
resolve,
|
|
);
|
|
});
|
|
return { code, runText };
|
|
};
|
|
|
|
it("exits 1 with the error on one line when the refresh fails", async () => {
|
|
const client = {
|
|
...fakeClient(),
|
|
collectionsSince: async () => {
|
|
throw new Error("HTTP 503 from server");
|
|
},
|
|
} as unknown as Client;
|
|
const dir = join(root, "backup");
|
|
const { code, runText } = await backupThroughRun(context(client), dir);
|
|
expect(code).toBe(1);
|
|
expect(runText).toBe("quak: HTTP 503 from server\n");
|
|
expect(stderr.text).toBe("Starting backup...\nRefreshing library...\n");
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
});
|
|
|
|
// A real saved session, read back by `loadSession`, whose server answers
|
|
// every request with 401, as it does once it no longer accepts the token.
|
|
// The context's prompts throw, so a prompt would end the run with another
|
|
// line and exit 1.
|
|
it("exits 3 with one line saying to log in again when the server answers 401", async () => {
|
|
const key = toBase64(new Uint8Array(32));
|
|
saveSession(join(root, "session"), {
|
|
email: "cli@example.com",
|
|
userID: USER_ID,
|
|
token: "expired",
|
|
masterKey: key,
|
|
secretKey: key,
|
|
publicKey: key,
|
|
});
|
|
const unauthorized = async (): Promise<Response> =>
|
|
new Response(null, { status: 401 });
|
|
const ctx = {
|
|
...context(),
|
|
loadSession: (path: string) =>
|
|
loadSession(path, { fetch: unauthorized }),
|
|
};
|
|
const dir = join(root, "backup");
|
|
const { code, runText } = await backupThroughRun(ctx, dir);
|
|
expect(code).toBe(3);
|
|
expect(runText).toBe(
|
|
`quak: the saved session is no longer valid; run "quak login"\n`,
|
|
);
|
|
expect(stderr.text).toBe("Starting backup...\nRefreshing library...\n");
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
});
|
|
|
|
it("exits 2 with one line naming the directory, sending no request, while another backup of it runs", async () => {
|
|
const dir = join(root, "backup");
|
|
// The lock another backup holds. Its modification time is set an hour
|
|
// ahead, so it stays current however long this test takes.
|
|
const lock = join(dir, "backup.lock");
|
|
mkdirSync(lock, { recursive: true });
|
|
const hourAhead = new Date(Date.now() + 3_600_000);
|
|
utimesSync(lock, hourAhead, hourAhead);
|
|
// A client whose refresh never finishes, so a run that started one
|
|
// before it exits would never return.
|
|
let requests = 0;
|
|
const never = (): Promise<never> => {
|
|
requests++;
|
|
return new Promise(() => {});
|
|
};
|
|
const client = {
|
|
...fakeClient(),
|
|
collectionsSince: never,
|
|
filesSince: never,
|
|
} as unknown as Client;
|
|
|
|
expect(await backupCommand(context(client), dir, {})).toBe(2);
|
|
expect(requests).toBe(0);
|
|
expect(stderr.text).toBe(
|
|
`Starting backup...\nquak: another backup of ${dir} is running\n`,
|
|
);
|
|
expect(readdirSync(dir)).toEqual(["backup.lock"]);
|
|
// Opening the library would have made its cache directory.
|
|
expect(existsSync(join(root, "cache"))).toBe(false);
|
|
});
|
|
|
|
// A backup runs for as long as its downloads take, hours on a large
|
|
// account. It lists its files from the refresh it waits for as it starts,
|
|
// so it must not ask the server for changes again on a timer while it
|
|
// runs: each refresh is a request, and each also starts a pass that asks
|
|
// again for the ML data of every file Ente holds none for.
|
|
//
|
|
// A saved session is read back by `loadSession`, its `fetch` a fake of the
|
|
// Ente HTTP API serving one album with one photo, encrypted as the server
|
|
// serves them. The fake holds the photo's download open while the clock
|
|
// moves on by two hours, then serves it. Only `setTimeout` and
|
|
// `clearTimeout` are faked: the library's refresh timer runs on them.
|
|
it("asks the server for changes only as it starts, however long it runs", async () => {
|
|
const masterKey = new Uint8Array(32).fill(1);
|
|
const albumKey = new Uint8Array(32).fill(2);
|
|
const photoKey = new Uint8Array(32).fill(3);
|
|
saveSession(join(root, "session"), {
|
|
email: "cli@example.com",
|
|
userID: USER_ID,
|
|
token: "token",
|
|
masterKey: toBase64(masterKey),
|
|
secretKey: toBase64(new Uint8Array(32)),
|
|
publicKey: toBase64(new Uint8Array(32)),
|
|
});
|
|
// A secretbox as the server sends one: ciphertext and nonce, base64.
|
|
const box = (plaintext: Uint8Array, key: Uint8Array) => {
|
|
const nonce = sodium.randombytes_buf(
|
|
sodium.crypto_secretbox_NONCEBYTES,
|
|
);
|
|
return {
|
|
ciphertext: toBase64(
|
|
sodium.crypto_secretbox_easy(plaintext, nonce, key),
|
|
),
|
|
nonce: toBase64(nonce),
|
|
};
|
|
};
|
|
const albumKeyBox = box(albumKey, masterKey);
|
|
const albumName = box(new TextEncoder().encode("Vacation"), albumKey);
|
|
const photoKeyBox = box(photoKey, albumKey);
|
|
const metadata = encryptBlob(
|
|
new TextEncoder().encode(
|
|
JSON.stringify({
|
|
title: "beach.jpg",
|
|
fileType: 0,
|
|
creationTime: 1_700_000_000_000_000,
|
|
modificationTime: 1_700_000_000_000_000,
|
|
}),
|
|
),
|
|
photoKey,
|
|
);
|
|
const content = encryptBlob(
|
|
new TextEncoder().encode("the photo's bytes"),
|
|
photoKey,
|
|
);
|
|
const album = {
|
|
id: 1,
|
|
owner: { id: USER_ID },
|
|
encryptedKey: albumKeyBox.ciphertext,
|
|
keyDecryptionNonce: albumKeyBox.nonce,
|
|
encryptedName: albumName.ciphertext,
|
|
nameDecryptionNonce: albumName.nonce,
|
|
type: "album",
|
|
updationTime: 1,
|
|
};
|
|
const photo = {
|
|
id: 100,
|
|
collectionID: 1,
|
|
ownerID: USER_ID,
|
|
encryptedKey: photoKeyBox.ciphertext,
|
|
keyDecryptionNonce: photoKeyBox.nonce,
|
|
metadata: {
|
|
encryptedData: toBase64(metadata.ciphertext),
|
|
decryptionHeader: toBase64(metadata.header),
|
|
},
|
|
file: { decryptionHeader: toBase64(content.header) },
|
|
// The backup fetches no thumbnail.
|
|
thumbnail: { decryptionHeader: "" },
|
|
updationTime: 1,
|
|
};
|
|
|
|
// Each refresh starts with `collectionsSince`, which sends one
|
|
// `/collections/v2` request.
|
|
let refreshes = 0;
|
|
let downloadRequested!: () => void;
|
|
const requested = new Promise<void>((resolve) => {
|
|
downloadRequested = resolve;
|
|
});
|
|
let releaseDownload!: () => void;
|
|
const released = new Promise<void>((resolve) => {
|
|
releaseDownload = resolve;
|
|
});
|
|
const json = (body: unknown): Response =>
|
|
new Response(JSON.stringify(body), {
|
|
headers: { "content-type": "application/json" },
|
|
});
|
|
const fetch = async (url: string | URL): Promise<Response> => {
|
|
const { hostname, pathname } = new URL(String(url));
|
|
if (hostname === "files.ente.io") {
|
|
downloadRequested();
|
|
await released;
|
|
return new Response(content.ciphertext);
|
|
}
|
|
if (pathname === "/collections/v2") {
|
|
refreshes++;
|
|
return json({ collections: [album] });
|
|
}
|
|
if (pathname === "/collections/v2/diff") {
|
|
return json({ diff: [photo], hasMore: false });
|
|
}
|
|
// Ente holds no ML data for the photo.
|
|
if (pathname === "/files/data/fetch") return json({ data: [] });
|
|
return new Response("not found", { status: 404 });
|
|
};
|
|
const ctx = {
|
|
...context(),
|
|
// The download's idle deadline is set past the two hours, or quak
|
|
// would give up the held download as stalled.
|
|
loadSession: (path: string) =>
|
|
loadSession(path, {
|
|
fetch: fetch as typeof globalThis.fetch,
|
|
downloadTimeoutMs: 3 * 3_600_000,
|
|
}),
|
|
};
|
|
|
|
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
|
try {
|
|
const backup = backupCommand(ctx, join(root, "backup"), {});
|
|
await requested;
|
|
vi.advanceTimersByTime(2 * 3_600_000);
|
|
releaseDownload();
|
|
expect(await backup).toBe(0);
|
|
} finally {
|
|
vi.useRealTimers();
|
|
}
|
|
|
|
// Two refreshes, both before the download: the library opens onto the
|
|
// empty cache with one, and the backup waits for the other before it
|
|
// lists its files.
|
|
expect(refreshes).toBe(2);
|
|
});
|
|
});
|
|
|
|
describe("helper list-missing-thumbnails", () => {
|
|
it("prints one line per file with an empty thumbnail", async () => {
|
|
const ctx = context(fakeClient({ emptyThumbID: 200 }));
|
|
expect(await listMissingThumbnailsCommand(ctx, {})).toBe(0);
|
|
expect(stdout.text).toBe(
|
|
"200\tdiagram.png\tWork\tempty thumbnail (0 bytes)\n",
|
|
);
|
|
expect(stderr.text).toContain("\n1 file(s) with missing thumbnails:\n");
|
|
});
|
|
|
|
it("says so when nothing is missing", async () => {
|
|
expect(await listMissingThumbnailsCommand(context(), {})).toBe(0);
|
|
expect(stdout.text).toBe("");
|
|
expect(stderr.text).toContain("No missing thumbnails found.\n");
|
|
});
|
|
|
|
// `--json` shapes stdout only: the progress stays on stderr, and the JSON
|
|
// takes the place of the count and the list.
|
|
it("prints a JSON array with --json, and its progress on stderr", async () => {
|
|
const ctx = context(fakeClient({ emptyThumbID: 200 }));
|
|
expect(await listMissingThumbnailsCommand(ctx, { json: true })).toBe(0);
|
|
expect(JSON.parse(stdout.text)).toEqual([
|
|
{
|
|
fileID: 200,
|
|
title: "diagram.png",
|
|
collection: "Work",
|
|
reason: "empty thumbnail (0 bytes)",
|
|
},
|
|
]);
|
|
expect(stderr.text).toContain(FETCHED_BOTH_ALBUMS);
|
|
expect(stderr.text).toContain("[Work] Checking thumbnails...\n");
|
|
expect(stderr.text).not.toContain("with missing thumbnails");
|
|
});
|
|
});
|
|
|
|
describe("helper fix-missing-thumbnails", () => {
|
|
// File 999 is in no album, so fixing it fails. The list of failed files
|
|
// is at error, and the last line is fatal, so `-q`, which prints fatal
|
|
// lines only, still says why the command exits 1.
|
|
it("lists a failed file at error, and ends with one fatal line counting the failures", async () => {
|
|
const last = `quak: 1 file(s) failed; see the "Failed files:" list, or the JSON with --json\n`;
|
|
const ctx = { ...context(), log: stderrLog(stderr, "error") };
|
|
expect(await fixMissingThumbnailsCommand(ctx, { file: ["999"] })).toBe(
|
|
1,
|
|
);
|
|
expect(stderr.text).toBe(
|
|
"\nFailed files:\n" +
|
|
" 999\tunknown\tfile not found in any collection\n" +
|
|
last,
|
|
);
|
|
|
|
stderr.text = "";
|
|
const quiet = { ...context(), log: stderrLog(stderr, "fatal") };
|
|
expect(
|
|
await fixMissingThumbnailsCommand(quiet, {
|
|
file: ["999"],
|
|
json: true,
|
|
}),
|
|
).toBe(1);
|
|
expect(stderr.text).toBe(last);
|
|
});
|
|
});
|
|
|
|
describe("backup-metadata --exif", () => {
|
|
// Runs the command and returns what it printed to stderr.
|
|
const backupMetadata = async (opts: { exif?: boolean; all?: boolean }) => {
|
|
expect(
|
|
await backupMetadataCommand(context(), join(root, "dump"), opts),
|
|
).toBe(0);
|
|
return stderr.text;
|
|
};
|
|
|
|
it("--exif extracts EXIF", async () => {
|
|
expect(await backupMetadata({ exif: true })).toContain(
|
|
"[beach.jpg] Extracting EXIF...\n",
|
|
);
|
|
});
|
|
|
|
it("--all extracts EXIF", async () => {
|
|
expect(await backupMetadata({ all: true })).toContain(
|
|
"[beach.jpg] Extracting EXIF...\n",
|
|
);
|
|
});
|
|
|
|
it("without either flag extracts no EXIF", async () => {
|
|
expect(await backupMetadata({})).not.toContain("Extracting EXIF");
|
|
});
|
|
});
|
|
|
|
// Each test first runs `files 1` so the cache holds Vacation as it was, then
|
|
// adds a file to Vacation on the server. (`collections` would not do: it reads
|
|
// the collection list alone and stores nothing.)
|
|
describe("backup-metadata and the thumbnail helpers refresh first", () => {
|
|
beforeEach(async () => {
|
|
expect(await filesCommand(context(), "1", {})).toBe(0);
|
|
stdout.text = "";
|
|
stderr.text = "";
|
|
});
|
|
|
|
it("backup-metadata writes a file added since the cache was written", async () => {
|
|
const ctx = context(fakeClient({ withNewFile: true }));
|
|
const dir = join(root, "dump");
|
|
expect(await backupMetadataCommand(ctx, dir, {})).toBe(0);
|
|
expect(
|
|
existsSync(join(dir, "collections", "1-Vacation", "102.json")),
|
|
).toBe(true);
|
|
});
|
|
|
|
it("list-missing-thumbnails checks a file added since the cache was written", async () => {
|
|
const ctx = context(
|
|
fakeClient({ withNewFile: true, emptyThumbID: 102 }),
|
|
);
|
|
expect(await listMissingThumbnailsCommand(ctx, {})).toBe(0);
|
|
expect(stdout.text).toBe(
|
|
"102\tnew.jpg\tVacation\tempty thumbnail (0 bytes)\n",
|
|
);
|
|
});
|
|
|
|
it("fix-missing-thumbnails finds a file added since the cache was written", async () => {
|
|
const ctx = context(fakeClient({ withNewFile: true }));
|
|
expect(
|
|
await fixMissingThumbnailsCommand(ctx, {
|
|
file: ["102"],
|
|
json: true,
|
|
}),
|
|
).toBe(0);
|
|
// Found, then skipped because the server records no thumbnail size
|
|
// for it; a file missing from the cache would fail as not found.
|
|
expect(JSON.parse(stdout.text)).toMatchObject([
|
|
{ fileID: 102, title: "new.jpg", status: "skipped" },
|
|
]);
|
|
});
|
|
|
|
// `run` in `cli-run.ts` prints a thrown error as one line and exits 1.
|
|
it("all three throw when the refresh fails", async () => {
|
|
const ctx = context(
|
|
fakeClient({ refreshError: "HTTP 503 from server" }),
|
|
);
|
|
const dir = join(root, "dump");
|
|
await expect(backupMetadataCommand(ctx, dir, {})).rejects.toThrow(
|
|
"HTTP 503 from server",
|
|
);
|
|
expect(existsSync(dir)).toBe(false);
|
|
await expect(listMissingThumbnailsCommand(ctx, {})).rejects.toThrow(
|
|
"HTTP 503 from server",
|
|
);
|
|
await expect(
|
|
fixMissingThumbnailsCommand(ctx, { file: ["100"] }),
|
|
).rejects.toThrow("HTTP 503 from server");
|
|
expect(stdout.text).toBe("");
|
|
});
|
|
});
|
|
|
|
// `files`, `get`, `get-thumb` and `helper fix-missing-thumbnails --file` take
|
|
// an ID only when it is written in decimal digits and is a safe integer.
|
|
// `Number()` alone reads `0x10` as 16, `1e3` as 1000 and an empty string as 0,
|
|
// so the command would act on a file the user never named.
|
|
// Anything else is refused with one line naming it, and exit 1, before any
|
|
// request.
|
|
//
|
|
// These run on a real client, read back from a saved session, over a fake Ente
|
|
// server holding an empty account, which records each request it receives. With
|
|
// a good ID the command goes on to ask the server, and finds nothing there.
|
|
describe("an ID given on the command line", () => {
|
|
const NOT_IDS = [
|
|
"0x10",
|
|
"1e3",
|
|
"1.5",
|
|
"-1",
|
|
"",
|
|
"abc",
|
|
// One past Number.MAX_SAFE_INTEGER: from here on a number cannot
|
|
// hold every integer, so two IDs could read as one.
|
|
"9007199254740992",
|
|
];
|
|
|
|
const requests: Request[] = [];
|
|
|
|
const emptyAccount = (): CliContext => {
|
|
const ctx = context();
|
|
const key = toBase64(new Uint8Array(32));
|
|
saveSession(ctx.sessionDir, {
|
|
email: "cli@example.com",
|
|
userID: USER_ID,
|
|
token: "token",
|
|
masterKey: key,
|
|
secretKey: key,
|
|
publicKey: key,
|
|
});
|
|
return {
|
|
...ctx,
|
|
loadSession: (path) =>
|
|
loadSession(path, {
|
|
fetch: async (url, init) => {
|
|
requests.push(new Request(url, init));
|
|
return new Response(
|
|
JSON.stringify({ collections: [] }),
|
|
{ headers: { "content-type": "application/json" } },
|
|
);
|
|
},
|
|
}),
|
|
};
|
|
};
|
|
|
|
beforeEach(() => {
|
|
requests.length = 0;
|
|
});
|
|
|
|
it("files looks up a decimal collection ID", async () => {
|
|
const ctx = emptyAccount();
|
|
expect(await filesCommand(ctx, "1", {})).toBe(1);
|
|
expect(stderr.text).toBe("Collection 1 not found\n");
|
|
expect(requests).not.toHaveLength(0);
|
|
});
|
|
|
|
it.each(NOT_IDS)("files refuses %j before any request", async (arg) => {
|
|
const ctx = emptyAccount();
|
|
expect(await filesCommand(ctx, arg, {})).toBe(1);
|
|
expect(stderr.text).toBe(`Invalid collection ID: ${arg}\n`);
|
|
expect(stdout.text).toBe("");
|
|
expect(requests).toHaveLength(0);
|
|
});
|
|
|
|
it("get looks up a decimal file ID", async () => {
|
|
expect(await getCommand(emptyAccount(), "100", {})).toBe(1);
|
|
expect(stderr.text).toBe("File 100 not found\n");
|
|
expect(requests).not.toHaveLength(0);
|
|
});
|
|
|
|
it.each(NOT_IDS)("get refuses %j before any request", async (arg) => {
|
|
expect(await getCommand(emptyAccount(), arg, {})).toBe(1);
|
|
expect(stderr.text).toBe(`Invalid file ID: ${arg}\n`);
|
|
expect(stdout.text).toBe("");
|
|
expect(requests).toHaveLength(0);
|
|
});
|
|
|
|
it("get-thumb looks up a decimal file ID", async () => {
|
|
expect(await getThumbCommand(emptyAccount(), "100", {})).toBe(1);
|
|
expect(stderr.text).toBe("File 100 not found\n");
|
|
expect(requests).not.toHaveLength(0);
|
|
});
|
|
|
|
it.each(NOT_IDS)("get-thumb refuses %j before any request", async (arg) => {
|
|
expect(await getThumbCommand(emptyAccount(), arg, {})).toBe(1);
|
|
expect(stderr.text).toBe(`Invalid file ID: ${arg}\n`);
|
|
expect(stdout.text).toBe("");
|
|
expect(requests).toHaveLength(0);
|
|
});
|
|
|
|
// The fixer reports each file it was given; on this account neither is
|
|
// found, which is a failure, so it exits 1.
|
|
it("fix-missing-thumbnails looks up each decimal --file ID", async () => {
|
|
const ctx = emptyAccount();
|
|
const opts = { file: ["100", "200"], json: true };
|
|
expect(await fixMissingThumbnailsCommand(ctx, opts)).toBe(1);
|
|
expect(JSON.parse(stdout.text)).toMatchObject([
|
|
{ fileID: 100, reason: "file not found in any collection" },
|
|
{ fileID: 200, reason: "file not found in any collection" },
|
|
]);
|
|
expect(requests).not.toHaveLength(0);
|
|
});
|
|
|
|
// One bad ID fails the whole command, so the good one before it is not
|
|
// fixed either.
|
|
it.each(NOT_IDS)(
|
|
"fix-missing-thumbnails refuses --file 100 %j before any request",
|
|
async (arg) => {
|
|
const ctx = emptyAccount();
|
|
const opts = { file: ["100", arg] };
|
|
expect(await fixMissingThumbnailsCommand(ctx, opts)).toBe(1);
|
|
expect(stderr.text).toBe(`Invalid file ID: ${arg}\n`);
|
|
expect(stdout.text).toBe("");
|
|
expect(requests).toHaveLength(0);
|
|
},
|
|
);
|
|
});
|
|
|
|
// An account with many albums (issue #210). On an empty cache `collections`
|
|
// used to wait for a refresh that fetched and decrypted the files of every
|
|
// album, one album after another, before it printed anything, so on such an
|
|
// account it seemed to hang. `files` with an album's ID did the same before it
|
|
// listed that album (issue #217). The account here has 40 albums of one photo
|
|
// each, encrypted as Ente's clients encrypt them, on the fake Ente server of
|
|
// `test/ente-server.ts`, which also records the path and query of each request
|
|
// quak sends. The session is the one logging in to that server saves.
|
|
describe("an account with many albums", () => {
|
|
const ALBUMS = 40;
|
|
const PASSWORD = "many-albums";
|
|
const paths: string[] = [];
|
|
let server: typeof globalThis.fetch;
|
|
let snapshot: ClientSnapshot;
|
|
|
|
// Album n is named `Album n` and holds one photo, n * 100, titled
|
|
// `photo-n.jpg`.
|
|
beforeAll(async () => {
|
|
await sodium.ready;
|
|
const account = await buildAccount(PASSWORD);
|
|
const collections: Record<string, unknown>[] = [];
|
|
const filesByCollection: Record<number, Record<string, unknown>[]> = {};
|
|
for (let id = 1; id <= ALBUMS; id++) {
|
|
const albumKey = sodium.crypto_secretbox_keygen();
|
|
const albumKeyBox = encryptSecretbox(albumKey, account.masterKey);
|
|
const name = encryptSecretbox(
|
|
new TextEncoder().encode(`Album ${id}`),
|
|
albumKey,
|
|
);
|
|
collections.push({
|
|
id,
|
|
owner: { id: USER_ID },
|
|
encryptedKey: toBase64(albumKeyBox.ciphertext),
|
|
keyDecryptionNonce: toBase64(albumKeyBox.nonce),
|
|
encryptedName: toBase64(name.ciphertext),
|
|
nameDecryptionNonce: toBase64(name.nonce),
|
|
type: "album",
|
|
updationTime: 1,
|
|
});
|
|
const photoKey =
|
|
sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const photoKeyBox = encryptSecretbox(photoKey, albumKey);
|
|
const metadata = encryptStreamBlob(
|
|
new TextEncoder().encode(
|
|
JSON.stringify({
|
|
title: `photo-${id}.jpg`,
|
|
fileType: 0,
|
|
creationTime: 1000,
|
|
modificationTime: 1000,
|
|
}),
|
|
),
|
|
photoKey,
|
|
);
|
|
filesByCollection[id] = [
|
|
{
|
|
id: id * 100,
|
|
collectionID: id,
|
|
ownerID: USER_ID,
|
|
encryptedKey: toBase64(photoKeyBox.ciphertext),
|
|
keyDecryptionNonce: toBase64(photoKeyBox.nonce),
|
|
metadata: {
|
|
encryptedData: toBase64(metadata.ciphertext),
|
|
decryptionHeader: toBase64(metadata.header),
|
|
},
|
|
// These tests fetch no original and no thumbnail.
|
|
file: { decryptionHeader: "" },
|
|
thumbnail: { decryptionHeader: "" },
|
|
updationTime: 1,
|
|
},
|
|
];
|
|
}
|
|
const ente = buildMetaFetch({
|
|
verifier: account.verifier,
|
|
srpAttributes: account.srpAttributes,
|
|
keyAttributes: account.keyAttributes,
|
|
encryptedToken: account.encryptedToken,
|
|
collections,
|
|
filesByCollection,
|
|
encryptedMLData: {},
|
|
fileCiphertexts: {},
|
|
});
|
|
server = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
|
const url =
|
|
typeof input === "string"
|
|
? input
|
|
: input instanceof URL
|
|
? input.href
|
|
: input.url;
|
|
const { pathname, search } = new URL(url);
|
|
paths.push(pathname + search);
|
|
return ente(input, init);
|
|
}) as typeof globalThis.fetch;
|
|
const client = await Client.login({
|
|
email: "many@example.com",
|
|
password: PASSWORD,
|
|
apiOptions: { fetch: server },
|
|
});
|
|
snapshot = client.toJSON();
|
|
});
|
|
|
|
beforeEach(() => {
|
|
paths.length = 0;
|
|
});
|
|
|
|
// A context whose saved session sends its requests to the fake server.
|
|
const accountContext = (): CliContext => {
|
|
const ctx = context();
|
|
saveSession(ctx.sessionDir, snapshot);
|
|
return {
|
|
...ctx,
|
|
loadSession: (path) => loadSession(path, { fetch: server }),
|
|
};
|
|
};
|
|
|
|
it("collections lists every album from one request, fetching no album's files", async () => {
|
|
const ctx = accountContext();
|
|
expect(await collectionsCommand(ctx, {})).toBe(0);
|
|
|
|
let albums = "";
|
|
for (let id = 1; id <= ALBUMS; id++)
|
|
albums += `${id}\talbum\tAlbum ${id}\n`;
|
|
expect(stdout.text).toBe(albums);
|
|
expect(stderr.text).toBe("");
|
|
expect(paths).toEqual(["/collections/v2?sinceTime=0"]);
|
|
|
|
// It stored nothing that would make a refresh skip an album's files:
|
|
// `files` after it still finds the album's photo.
|
|
stdout.text = "";
|
|
expect(await filesCommand(ctx, "7", {})).toBe(0);
|
|
expect(stdout.text).toBe("700\timage\tphoto-7.jpg\n");
|
|
});
|
|
|
|
it("files fetches the files of the album it lists and of no other", async () => {
|
|
expect(await filesCommand(accountContext(), "7", {})).toBe(0);
|
|
|
|
expect(stdout.text).toBe("700\timage\tphoto-7.jpg\n");
|
|
expect(stderr.text).toBe("");
|
|
expect(paths).toEqual([
|
|
"/collections/v2?sinceTime=0",
|
|
"/collections/v2/diff?collectionID=7&sinceTime=0",
|
|
]);
|
|
});
|
|
|
|
// The first listing stored the album, so the second finds its stored copy
|
|
// current and asks for no album's files.
|
|
it("files run again for the same album sends no request for its files", async () => {
|
|
const ctx = accountContext();
|
|
expect(await filesCommand(ctx, "7", {})).toBe(0);
|
|
stdout.text = "";
|
|
paths.length = 0;
|
|
|
|
expect(await filesCommand(ctx, "7", {})).toBe(0);
|
|
expect(stdout.text).toBe("700\timage\tphoto-7.jpg\n");
|
|
expect(paths).toEqual(["/collections/v2?sinceTime=0"]);
|
|
});
|
|
|
|
it("files with no album's ID counts on stderr each album whose files its refresh fetched", async () => {
|
|
expect(await filesCommand(accountContext(), undefined, {})).toBe(0);
|
|
|
|
let photos = "";
|
|
let progress = "";
|
|
for (let n = 1; n <= ALBUMS; n++) {
|
|
photos += `${n * 100}\timage\tphoto-${n}.jpg\n`;
|
|
progress += `Fetched the files of ${n} of ${ALBUMS} collections\n`;
|
|
}
|
|
expect(stdout.text).toBe(photos);
|
|
expect(stderr.text).toBe(progress);
|
|
});
|
|
|
|
// `--json` shapes stdout only, so the refresh's progress is still counted
|
|
// on stderr.
|
|
it("files --json with no album's ID counts on stderr each album whose files its refresh fetched", async () => {
|
|
expect(
|
|
await filesCommand(accountContext(), undefined, { json: true }),
|
|
).toBe(0);
|
|
|
|
const rows = JSON.parse(stdout.text);
|
|
expect(rows).toHaveLength(ALBUMS);
|
|
expect(rows[6]).toEqual({
|
|
id: 700,
|
|
title: "photo-7.jpg",
|
|
fileType: "image",
|
|
creationTime: 1000,
|
|
collectionID: 7,
|
|
});
|
|
let progress = "";
|
|
for (let n = 1; n <= ALBUMS; n++) {
|
|
progress += `Fetched the files of ${n} of ${ALBUMS} collections\n`;
|
|
}
|
|
expect(stderr.text).toBe(progress);
|
|
});
|
|
});
|