sneak
0cbe338b58
Tests only; the retry module they import does not exist yet, so the branch is red at this commit. New test/retry/retry.test.ts documents the classifier and the backoff: which errors are worth another attempt, which are not, and how the delay before each retry is derived. It asserts on the arguments handed to an injected sleep function rather than on elapsed time, so the suite never waits and the numbers are exact. test/api/client.test.ts gains the request-count contract for each of the six call sites, the deadline behaviour, the ApiError typing that the presigned PUT and the null-body paths need in order to be classified at all, and the replay rule for the two non-idempotent methods. test/download/download.test.ts gains the case that motivates the whole design: a socket reset after the response headers arrived, which happens below ApiClient and can only be caught by retrying the request, the stream consumption and the decryption together. It also pins that a retried download stages exactly one temp file, and that the two retry layers do not compose into a multiplied request budget. The existing truncation tests now assert on the error type rather than its wording, since that type is what the classifier reads. test/thumbnails/thumbnails.test.ts separates a genuine 404 from an exhausted retry, so a failing server can no longer make fix-missing-thumbnails re-upload thumbnails that already exist.
470 lines
16 KiB
TypeScript
470 lines
16 KiB
TypeScript
/**
|
|
* Tests for the `quak backup` command's core logic.
|
|
*
|
|
* `quak backup <dir>` downloads every file from every collection into
|
|
* a local directory tree:
|
|
*
|
|
* <dir>/
|
|
* <collection-name>/
|
|
* <file-title>
|
|
* <file-title>
|
|
* <collection-name>/
|
|
* ...
|
|
* metadata.json (all decrypted collection + file metadata)
|
|
*
|
|
* The backup command has two properties that distinguish it from a naive
|
|
* "download everything" loop:
|
|
*
|
|
* 1. **Skip existing files.** If `<dir>/<collection>/<title>` already
|
|
* exists on disk and its size matches the decrypted content length
|
|
* recorded in metadata.json from a prior run, the file is not
|
|
* re-downloaded. This makes interrupted backups resumable and
|
|
* incremental runs fast.
|
|
*
|
|
* 2. **Never crash on a single file failure.** If a file download or
|
|
* decryption fails, the error is logged and the backup continues
|
|
* with the next file. At the end, the exit code is non-zero if any
|
|
* files failed, and the summary lists them. The Ente first-party
|
|
* CLI crashes entirely when a single file can't be retrieved,
|
|
* which defeats the purpose of a backup tool.
|
|
*
|
|
* These tests exercise the backup logic (in src/backup.ts) using the
|
|
* same mock server from the Client usage tests. The CLI binary itself
|
|
* is a thin wrapper around this module.
|
|
*/
|
|
|
|
import {
|
|
existsSync,
|
|
lstatSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
readlinkSync,
|
|
rmSync,
|
|
} from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import sodium from "libsodium-wrappers-sumo";
|
|
import { SRP, SrpServer } from "fast-srp-hap";
|
|
import { beforeAll, afterAll, describe, expect, it } from "vitest";
|
|
import {
|
|
init,
|
|
toBase64,
|
|
deriveKEK,
|
|
deriveLoginSubkey,
|
|
} from "../../src/crypto/index.js";
|
|
import { Client } from "../../src/client.js";
|
|
import { runBackup } from "../../src/backup.js";
|
|
import type { KeyAttributes } from "../../src/auth/types.js";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Mock server (condensed from usage.test.ts)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const TEST_EMAIL = "backup@example.com";
|
|
const TEST_PASSWORD = "backuppass";
|
|
const TEST_OPS = 2;
|
|
const TEST_MEM = 64 * 1024 * 1024;
|
|
|
|
interface MockState {
|
|
verifier: Buffer;
|
|
srpAttributes: Record<string, unknown>;
|
|
keyAttributes: KeyAttributes;
|
|
encryptedToken: string;
|
|
collections: Record<string, unknown>[];
|
|
files: Record<
|
|
number,
|
|
{ raw: Record<string, unknown>; plaintext: Uint8Array }
|
|
>;
|
|
}
|
|
|
|
let mock: MockState;
|
|
let testDir: string;
|
|
|
|
const buildMock = async (): Promise<MockState> => {
|
|
const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
|
|
const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM);
|
|
const loginSubKeyBytes = deriveLoginSubkey(kek);
|
|
|
|
const srpUserID = "backup-srp";
|
|
const srpSalt = sodium.randombytes_buf(16);
|
|
const verifier = SRP.computeVerifier(
|
|
SRP.params["4096"],
|
|
Buffer.from(srpSalt),
|
|
Buffer.from(srpUserID),
|
|
Buffer.from(loginSubKeyBytes),
|
|
);
|
|
|
|
const masterKey = sodium.randombytes_buf(32);
|
|
const keyNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encryptedKey = sodium.crypto_secretbox_easy(masterKey, keyNonce, kek);
|
|
const kp = sodium.crypto_box_keypair();
|
|
const skNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encSK = sodium.crypto_secretbox_easy(
|
|
kp.privateKey,
|
|
skNonce,
|
|
masterKey,
|
|
);
|
|
const tokenBytes = sodium.randombytes_buf(32);
|
|
const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey);
|
|
|
|
const keyAttributes: KeyAttributes = {
|
|
kekSalt: toBase64(kekSalt),
|
|
encryptedKey: toBase64(encryptedKey),
|
|
keyDecryptionNonce: toBase64(keyNonce),
|
|
publicKey: toBase64(kp.publicKey),
|
|
encryptedSecretKey: toBase64(encSK),
|
|
secretKeyDecryptionNonce: toBase64(skNonce),
|
|
memLimit: TEST_MEM,
|
|
opsLimit: TEST_OPS,
|
|
};
|
|
|
|
const makeCollection = (id: number, name: string) => {
|
|
const ck = sodium.crypto_secretbox_keygen();
|
|
const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encCK = sodium.crypto_secretbox_easy(ck, ckN, masterKey);
|
|
const nameBytes = new TextEncoder().encode(name);
|
|
const cnN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encCN = sodium.crypto_secretbox_easy(nameBytes, cnN, ck);
|
|
return {
|
|
raw: {
|
|
id,
|
|
owner: { id: 42 },
|
|
encryptedKey: toBase64(encCK),
|
|
keyDecryptionNonce: toBase64(ckN),
|
|
encryptedName: toBase64(encCN),
|
|
nameDecryptionNonce: toBase64(cnN),
|
|
type: "album",
|
|
updationTime: 1700000000000000,
|
|
},
|
|
key: ck,
|
|
};
|
|
};
|
|
|
|
const makeFile = (
|
|
id: number,
|
|
collKey: Uint8Array,
|
|
title: string,
|
|
plaintext: Uint8Array,
|
|
collID: number,
|
|
) => {
|
|
const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey);
|
|
|
|
const meta = JSON.stringify({
|
|
title,
|
|
fileType: 0,
|
|
creationTime: 1700000000000000,
|
|
modificationTime: 1700000000000000,
|
|
});
|
|
const metaPush =
|
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
|
const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push(
|
|
metaPush.state,
|
|
new TextEncoder().encode(meta),
|
|
null,
|
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
|
);
|
|
|
|
const filePush =
|
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
|
const encFile = sodium.crypto_secretstream_xchacha20poly1305_push(
|
|
filePush.state,
|
|
plaintext,
|
|
null,
|
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
|
);
|
|
|
|
return {
|
|
raw: {
|
|
id,
|
|
collectionID: collID,
|
|
ownerID: 42,
|
|
encryptedKey: toBase64(encFK),
|
|
keyDecryptionNonce: toBase64(fkN),
|
|
metadata: {
|
|
encryptedData: toBase64(encMeta),
|
|
decryptionHeader: toBase64(metaPush.header),
|
|
},
|
|
file: { decryptionHeader: toBase64(filePush.header) },
|
|
thumbnail: {
|
|
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
|
},
|
|
updationTime: 1700000000000000,
|
|
},
|
|
plaintext,
|
|
ciphertext: encFile,
|
|
};
|
|
};
|
|
|
|
const col1 = makeCollection(1, "Vacation");
|
|
const col2 = makeCollection(2, "Work");
|
|
const file1 = makeFile(
|
|
100,
|
|
col1.key,
|
|
"beach.jpg",
|
|
sodium.randombytes_buf(3000),
|
|
1,
|
|
);
|
|
const file2 = makeFile(
|
|
101,
|
|
col1.key,
|
|
"sunset.jpg",
|
|
sodium.randombytes_buf(2000),
|
|
1,
|
|
);
|
|
const file3 = makeFile(
|
|
200,
|
|
col2.key,
|
|
"diagram.png",
|
|
sodium.randombytes_buf(1500),
|
|
2,
|
|
);
|
|
|
|
return {
|
|
verifier,
|
|
srpAttributes: {
|
|
srpUserID,
|
|
srpSalt: toBase64(srpSalt),
|
|
memLimit: TEST_MEM,
|
|
opsLimit: TEST_OPS,
|
|
kekSalt: toBase64(kekSalt),
|
|
isEmailMFAEnabled: false,
|
|
},
|
|
keyAttributes,
|
|
encryptedToken: toBase64(encToken),
|
|
collections: [col1.raw, col2.raw],
|
|
files: {
|
|
1: {
|
|
raw: [file1.raw, file2.raw],
|
|
ciphertexts: { 100: file1.ciphertext, 101: file2.ciphertext },
|
|
},
|
|
2: { raw: [file3.raw], ciphertexts: { 200: file3.ciphertext } },
|
|
100: { plaintext: file1.plaintext },
|
|
101: { plaintext: file2.plaintext },
|
|
200: { plaintext: file3.plaintext },
|
|
} as Record<number, unknown>,
|
|
};
|
|
};
|
|
|
|
const buildMockFetch = (m: MockState, opts?: { failFileID?: number }) => {
|
|
let srpServer: SrpServer;
|
|
|
|
return (async (
|
|
input: RequestInfo | URL,
|
|
init?: RequestInit,
|
|
): Promise<Response> => {
|
|
const url =
|
|
typeof input === "string"
|
|
? input
|
|
: input instanceof URL
|
|
? input.href
|
|
: input.url;
|
|
const parsed = new URL(url);
|
|
const path = parsed.pathname;
|
|
const json = (body: unknown) =>
|
|
new Response(JSON.stringify(body), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
});
|
|
|
|
if (path === "/users/srp/attributes")
|
|
return json({ attributes: m.srpAttributes });
|
|
|
|
if (path === "/users/srp/create-session") {
|
|
const body = JSON.parse(init?.body as string);
|
|
const serverKey = await SRP.genKey();
|
|
srpServer = new SrpServer(
|
|
SRP.params["4096"],
|
|
m.verifier,
|
|
serverKey,
|
|
);
|
|
const B = srpServer.computeB();
|
|
srpServer.setA(Buffer.from(body.srpA, "base64"));
|
|
return json({ sessionID: "s1", srpB: B.toString("base64") });
|
|
}
|
|
|
|
if (path === "/users/srp/verify-session") {
|
|
const body = JSON.parse(init?.body as string);
|
|
srpServer.checkM1(Buffer.from(body.srpM1, "base64"));
|
|
return json({
|
|
srpM2: srpServer.computeM2().toString("base64"),
|
|
id: 42,
|
|
keyAttributes: m.keyAttributes,
|
|
encryptedToken: m.encryptedToken,
|
|
});
|
|
}
|
|
|
|
if (path === "/collections/v2")
|
|
return json({ collections: m.collections });
|
|
|
|
if (path === "/collections/v2/diff") {
|
|
const collID = Number(parsed.searchParams.get("collectionID"));
|
|
const collData = m.files[collID] as { raw: unknown[] } | undefined;
|
|
return json({ diff: collData?.raw ?? [], hasMore: false });
|
|
}
|
|
|
|
// File download
|
|
if (url.includes("fileID=") || path.startsWith("/files/download/")) {
|
|
const fileID = Number(
|
|
parsed.searchParams.get("fileID") ?? path.split("/").pop(),
|
|
);
|
|
if (opts?.failFileID === fileID) {
|
|
return new Response("Internal Server Error", { status: 500 });
|
|
}
|
|
const collData = Object.values(m.files).find(
|
|
(v: unknown) =>
|
|
v &&
|
|
typeof v === "object" &&
|
|
"ciphertexts" in (v as Record<string, unknown>) &&
|
|
fileID in
|
|
(v as Record<string, Record<number, unknown>>)
|
|
.ciphertexts,
|
|
) as { ciphertexts: Record<number, Uint8Array> } | undefined;
|
|
if (collData) {
|
|
return new Response(collData.ciphertexts[fileID], {
|
|
status: 200,
|
|
});
|
|
}
|
|
return new Response("not found", { status: 404 });
|
|
}
|
|
|
|
return new Response("not found", { status: 404 });
|
|
}) as typeof globalThis.fetch;
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Setup / teardown
|
|
// ---------------------------------------------------------------------------
|
|
|
|
beforeAll(async () => {
|
|
await init();
|
|
await sodium.ready;
|
|
mock = await buildMock();
|
|
testDir = mkdtempSync(join(tmpdir(), "quak-backup-test-"));
|
|
});
|
|
|
|
afterAll(() => {
|
|
if (testDir && existsSync(testDir))
|
|
rmSync(testDir, { recursive: true, force: true });
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe("quak backup", () => {
|
|
it("downloads all files organized by collection name", async () => {
|
|
const outDir = join(testDir, "full-backup");
|
|
const client = await Client.login({
|
|
email: TEST_EMAIL,
|
|
password: TEST_PASSWORD,
|
|
apiOptions: { fetch: buildMockFetch(mock) },
|
|
});
|
|
|
|
const result = await runBackup(client, outDir);
|
|
|
|
expect(result.totalFiles).toBe(3);
|
|
expect(result.downloaded).toBe(3);
|
|
expect(result.skipped).toBe(0);
|
|
expect(result.failed).toBe(0);
|
|
expect(result.errors).toEqual([]);
|
|
|
|
// Originals are under <outDir>/originals/<fileID>.<ext>
|
|
expect(readFileSync(join(outDir, "originals", "100.jpg")).length).toBe(
|
|
3000,
|
|
);
|
|
expect(readFileSync(join(outDir, "originals", "101.jpg")).length).toBe(
|
|
2000,
|
|
);
|
|
expect(readFileSync(join(outDir, "originals", "200.png")).length).toBe(
|
|
1500,
|
|
);
|
|
|
|
// Collection dirs under collections/ contain symlinks to originals
|
|
const beachLink = join(outDir, "collections", "Vacation", "beach.jpg");
|
|
expect(lstatSync(beachLink).isSymbolicLink()).toBe(true);
|
|
expect(readlinkSync(beachLink)).toContain("originals");
|
|
expect(readFileSync(beachLink).length).toBe(3000);
|
|
});
|
|
|
|
it("skips files that already exist on disk with matching size", async () => {
|
|
const outDir = join(testDir, "incremental");
|
|
const client = await Client.login({
|
|
email: TEST_EMAIL,
|
|
password: TEST_PASSWORD,
|
|
apiOptions: { fetch: buildMockFetch(mock) },
|
|
});
|
|
|
|
// First run: download everything
|
|
const first = await runBackup(client, outDir);
|
|
expect(first.downloaded).toBe(3);
|
|
|
|
// Second run: everything should be skipped
|
|
const second = await runBackup(client, outDir);
|
|
expect(second.downloaded).toBe(0);
|
|
expect(second.skipped).toBe(3);
|
|
expect(second.failed).toBe(0);
|
|
});
|
|
|
|
it("continues after a single file download failure", async () => {
|
|
// File 101 (sunset.jpg) will return HTTP 500. The other two
|
|
// files must still download. The result must report the failure
|
|
// without throwing.
|
|
//
|
|
// A 500 is retryable, so this file now costs several requests before
|
|
// it is given up on — that is the point of the retry policy, and
|
|
// `runBackup`'s own resilience is unchanged by it: the retry lives
|
|
// strictly below this loop, and an exhausted file is still logged,
|
|
// counted, and stepped over rather than aborting the run. The
|
|
// injected `sleep` is what keeps the suite from actually waiting out
|
|
// the backoff.
|
|
const outDir = join(testDir, "partial-failure");
|
|
const client = await Client.login({
|
|
email: TEST_EMAIL,
|
|
password: TEST_PASSWORD,
|
|
apiOptions: {
|
|
fetch: buildMockFetch(mock, { failFileID: 101 }),
|
|
retry: { sleep: () => Promise.resolve(), random: () => 0 },
|
|
},
|
|
});
|
|
|
|
const result = await runBackup(client, outDir);
|
|
|
|
expect(result.totalFiles).toBe(3);
|
|
expect(result.downloaded).toBe(2);
|
|
expect(result.failed).toBe(1);
|
|
expect(result.errors.length).toBe(1);
|
|
expect(result.errors[0]!.fileID).toBe(101);
|
|
expect(result.errors[0]!.title).toBe("sunset.jpg");
|
|
|
|
// The two successful originals are on disk
|
|
expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true);
|
|
expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true);
|
|
// The failed file has no original and no symlink
|
|
expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(false);
|
|
expect(
|
|
existsSync(join(outDir, "collections", "Vacation", "sunset.jpg")),
|
|
).toBe(false);
|
|
});
|
|
|
|
it("writes per-collection JSON metadata", async () => {
|
|
const outDir = join(testDir, "metadata-check");
|
|
const client = await Client.login({
|
|
email: TEST_EMAIL,
|
|
password: TEST_PASSWORD,
|
|
apiOptions: { fetch: buildMockFetch(mock) },
|
|
});
|
|
|
|
await runBackup(client, outDir);
|
|
|
|
// Each collection gets a <name>.json next to its image dir
|
|
const vacationMeta = join(outDir, "collections", "Vacation.json");
|
|
expect(existsSync(vacationMeta)).toBe(true);
|
|
const meta = JSON.parse(readFileSync(vacationMeta, "utf-8"));
|
|
expect(meta.name).toBe("Vacation");
|
|
expect(meta.files.length).toBeGreaterThan(0);
|
|
expect(meta.files[0].metadata.title).toBeDefined();
|
|
});
|
|
});
|