clawbot
cda57eebda
check / check (push) Successful in 49s
The server accepts a new thumbnail only from the file's owner and only when it is no larger than the thumbnail size it records. Both thumbnail helpers now skip files another account owns without fetching them. The fixer skips a file whose recorded thumbnail size is 0 or unknown before downloading it, and otherwise tries smaller encodings until the encrypted thumbnail fits, skipping the file if none does. Model: opus-5-5
852 lines
32 KiB
TypeScript
852 lines
32 KiB
TypeScript
/**
|
|
* Tests for `listMissingThumbnails` and `fixMissingThumbnails`.
|
|
*
|
|
* These use a mock Ente server that serves encrypted collections, files,
|
|
* and thumbnails. The mock server has deliberate gaps: some files have
|
|
* working thumbnails, others return 404 or empty bodies. The tests
|
|
* verify that the detection and repair logic handles each case correctly.
|
|
*
|
|
* As of issue #52 both helpers take an open `Library` for enumeration and the
|
|
* `Client` for the API operations that stay unchanged (the thumbnail existence
|
|
* check, and the encrypt-and-upload path). `fixMissingThumbnails` reads each
|
|
* original through the library's content cache (`photo.original()`).
|
|
*
|
|
* `fixMissingThumbnails` is the most complex function in quak: it
|
|
* downloads the original file, generates a JPEG thumbnail with jpeg-js,
|
|
* encrypts it with secretstream push, gets a presigned upload URL,
|
|
* uploads to S3, and registers the new thumbnail with the API. The
|
|
* test verifies each step actually happened and the uploaded data is
|
|
* a valid encrypted blob that decrypts to a JPEG.
|
|
*
|
|
* It regenerates thumbnails for baseline JPEGs only, because `jpeg-js` decodes
|
|
* only JPEG. A non-JPEG image (PNG, HEIC) or a video is reported as "skipped
|
|
* (unsupported)" rather than crashing the decoder into an opaque failure
|
|
* (issue #17); the mixed test below locks that distinction down.
|
|
*/
|
|
|
|
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import sodium from "libsodium-wrappers-sumo";
|
|
import * as jpegJs from "jpeg-js";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import {
|
|
init,
|
|
toBase64,
|
|
decryptBlob,
|
|
fromBase64,
|
|
deriveKEK,
|
|
deriveLoginSubkey,
|
|
} from "../../src/crypto/index.js";
|
|
import { SRP, SrpServer } from "fast-srp-hap";
|
|
import { Client } from "../../src/client.js";
|
|
import { Library } from "../../src/library/index.js";
|
|
import {
|
|
listMissingThumbnails,
|
|
fixMissingThumbnails,
|
|
} from "../../src/thumbnails.js";
|
|
import type { KeyAttributes } from "../../src/auth/types.js";
|
|
import type { RetryOptions } from "../../src/retry.js";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Mock server with controllable thumbnail behavior
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const TEST_EMAIL = "thumb@example.com";
|
|
const TEST_PASSWORD = "thumbpass";
|
|
const TEST_OPS = 2;
|
|
const TEST_MEM = 64 * 1024 * 1024;
|
|
const TEST_TIME = 1700000000000000;
|
|
|
|
interface ThumbMockState {
|
|
verifier: Buffer;
|
|
srpAttributes: Record<string, unknown>;
|
|
keyAttributes: KeyAttributes;
|
|
encryptedToken: string;
|
|
collections: Record<string, unknown>[];
|
|
filesByCollection: Record<number, Record<string, unknown>[]>;
|
|
fileCiphertexts: Record<number, Uint8Array>;
|
|
fileKeys: Record<number, Uint8Array>;
|
|
thumbnailBehavior: Record<number, "ok" | "empty" | "404" | "500">;
|
|
// Captures from fix operations
|
|
uploadedThumbnails: {
|
|
fileID: number;
|
|
objectKey: string;
|
|
decryptionHeader: string;
|
|
ciphertext: Uint8Array;
|
|
}[];
|
|
}
|
|
|
|
let mock: ThumbMockState;
|
|
let tmpRoot: string;
|
|
|
|
// PNG signature bytes — enough for `fixMissingThumbnails` to recognise a
|
|
// non-JPEG image and skip it. It need not be a decodable PNG.
|
|
const PNG_BYTES = new Uint8Array([
|
|
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
|
|
]);
|
|
|
|
const buildThumbMock = async (opts?: {
|
|
extraFormats?: boolean;
|
|
}): Promise<ThumbMockState> => {
|
|
const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
|
|
const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM);
|
|
const loginSubKeyBytes = deriveLoginSubkey(kek);
|
|
|
|
const srpUserID = "thumb-srp";
|
|
const srpSalt = sodium.randombytes_buf(16);
|
|
const verifier = SRP.computeVerifier(
|
|
SRP.params["4096"],
|
|
Buffer.from(srpSalt),
|
|
Buffer.from(srpUserID),
|
|
Buffer.from(loginSubKeyBytes),
|
|
);
|
|
|
|
const masterKey = sodium.randombytes_buf(32);
|
|
const keyNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encryptedKey = sodium.crypto_secretbox_easy(masterKey, keyNonce, kek);
|
|
const kp = sodium.crypto_box_keypair();
|
|
const skNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encSK = sodium.crypto_secretbox_easy(
|
|
kp.privateKey,
|
|
skNonce,
|
|
masterKey,
|
|
);
|
|
const tokenBytes = sodium.randombytes_buf(32);
|
|
const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey);
|
|
|
|
const keyAttributes: KeyAttributes = {
|
|
kekSalt: toBase64(kekSalt),
|
|
encryptedKey: toBase64(encryptedKey),
|
|
keyDecryptionNonce: toBase64(keyNonce),
|
|
publicKey: toBase64(kp.publicKey),
|
|
encryptedSecretKey: toBase64(encSK),
|
|
secretKeyDecryptionNonce: toBase64(skNonce),
|
|
memLimit: TEST_MEM,
|
|
opsLimit: TEST_OPS,
|
|
};
|
|
|
|
const collKey = sodium.crypto_secretbox_keygen();
|
|
const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encCK = sodium.crypto_secretbox_easy(collKey, ckN, masterKey);
|
|
const nameB = new TextEncoder().encode("Photos");
|
|
const cnN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encCN = sodium.crypto_secretbox_easy(nameB, cnN, collKey);
|
|
|
|
const rawCollection = {
|
|
id: 1,
|
|
owner: { id: 42 },
|
|
encryptedKey: toBase64(encCK),
|
|
keyDecryptionNonce: toBase64(ckN),
|
|
encryptedName: toBase64(encCN),
|
|
nameDecryptionNonce: toBase64(cnN),
|
|
type: "album",
|
|
updationTime: TEST_TIME,
|
|
};
|
|
|
|
// Generate a real tiny JPEG via jpeg-js, used as the encrypted body of the
|
|
// JPEG files so a repair actually decodes and re-encodes real pixels.
|
|
const w = 100;
|
|
const h = 80;
|
|
const pixels = new Uint8Array(w * h * 4);
|
|
for (let i = 0; i < pixels.length; i += 4) {
|
|
pixels[i] = 255; // R
|
|
pixels[i + 1] = 0; // G
|
|
pixels[i + 2] = 0; // B
|
|
pixels[i + 3] = 255; // A
|
|
}
|
|
const tinyJpeg = new Uint8Array(
|
|
jpegJs.encode({ data: pixels, width: w, height: h }, 80).data,
|
|
);
|
|
|
|
const fileKeys: Record<number, Uint8Array> = {};
|
|
const fileCiphertexts: Record<number, Uint8Array> = {};
|
|
|
|
// Build one raw file record: encrypt its metadata and its body under a
|
|
// fresh per-file key, and record the key and ciphertext for the mock to
|
|
// serve and for the test to verify against.
|
|
const makeRawFile = (
|
|
fileID: number,
|
|
fileType: number,
|
|
title: string,
|
|
body: Uint8Array,
|
|
): Record<string, unknown> => {
|
|
const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
fileKeys[fileID] = fk;
|
|
const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey);
|
|
|
|
const meta = JSON.stringify({
|
|
title,
|
|
fileType,
|
|
creationTime: TEST_TIME,
|
|
modificationTime: TEST_TIME,
|
|
});
|
|
const metaPush =
|
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
|
const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push(
|
|
metaPush.state,
|
|
new TextEncoder().encode(meta),
|
|
null,
|
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
|
);
|
|
|
|
const filePush =
|
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
|
const encFile = sodium.crypto_secretstream_xchacha20poly1305_push(
|
|
filePush.state,
|
|
body,
|
|
null,
|
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
|
);
|
|
fileCiphertexts[fileID] = encFile;
|
|
|
|
return {
|
|
id: fileID,
|
|
collectionID: 1,
|
|
ownerID: 42,
|
|
encryptedKey: toBase64(encFK),
|
|
keyDecryptionNonce: toBase64(fkN),
|
|
metadata: {
|
|
encryptedData: toBase64(encMeta),
|
|
decryptionHeader: toBase64(metaPush.header),
|
|
},
|
|
file: { decryptionHeader: toBase64(filePush.header) },
|
|
thumbnail: {
|
|
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
|
},
|
|
// The encrypted size of the thumbnail the server records; large
|
|
// enough here that the default encoding fits.
|
|
info: { thumbSize: 1_000_000 },
|
|
updationTime: TEST_TIME,
|
|
};
|
|
};
|
|
|
|
// Three JPEG files: ok thumbnail, empty thumbnail, 404 thumbnail.
|
|
const rawFiles: Record<string, unknown>[] = [];
|
|
for (const fileID of [100, 101, 102]) {
|
|
rawFiles.push(makeRawFile(fileID, 0, `file-${fileID}.jpg`, tinyJpeg));
|
|
}
|
|
const thumbnailBehavior: Record<number, "ok" | "empty" | "404" | "500"> = {
|
|
100: "ok",
|
|
101: "empty",
|
|
102: "404",
|
|
};
|
|
|
|
// For the issue #17 mixed test: a non-JPEG image and a video, both with a
|
|
// missing (404) thumbnail so they surface in the missing list too.
|
|
if (opts?.extraFormats) {
|
|
rawFiles.push(makeRawFile(103, 0, "file-103.png", PNG_BYTES));
|
|
rawFiles.push(
|
|
makeRawFile(104, 1, "file-104.mp4", new Uint8Array([0, 0, 0, 1])),
|
|
);
|
|
thumbnailBehavior[103] = "404";
|
|
thumbnailBehavior[104] = "404";
|
|
}
|
|
|
|
return {
|
|
verifier,
|
|
srpAttributes: {
|
|
srpUserID,
|
|
srpSalt: toBase64(srpSalt),
|
|
memLimit: TEST_MEM,
|
|
opsLimit: TEST_OPS,
|
|
kekSalt: toBase64(kekSalt),
|
|
isEmailMFAEnabled: false,
|
|
},
|
|
keyAttributes,
|
|
encryptedToken: toBase64(encToken),
|
|
collections: [rawCollection],
|
|
filesByCollection: { 1: rawFiles },
|
|
fileCiphertexts,
|
|
fileKeys,
|
|
thumbnailBehavior,
|
|
uploadedThumbnails: [],
|
|
};
|
|
};
|
|
|
|
const buildThumbFetch = (m: ThumbMockState) => {
|
|
let srpServer: SrpServer;
|
|
|
|
return (async (
|
|
input: RequestInfo | URL,
|
|
init?: RequestInit,
|
|
): Promise<Response> => {
|
|
const url =
|
|
typeof input === "string"
|
|
? input
|
|
: input instanceof URL
|
|
? input.href
|
|
: input.url;
|
|
const parsed = new URL(url);
|
|
const path = parsed.pathname;
|
|
const json = (body: unknown) =>
|
|
new Response(JSON.stringify(body), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
});
|
|
|
|
// SRP auth flow
|
|
if (path === "/users/srp/attributes")
|
|
return json({ attributes: m.srpAttributes });
|
|
if (path === "/users/srp/create-session") {
|
|
const body = JSON.parse(init?.body as string);
|
|
const serverKey = await SRP.genKey();
|
|
srpServer = new SrpServer(
|
|
SRP.params["4096"],
|
|
m.verifier,
|
|
serverKey,
|
|
);
|
|
const B = srpServer.computeB();
|
|
srpServer.setA(Buffer.from(body.srpA, "base64"));
|
|
return json({ sessionID: "s1", srpB: B.toString("base64") });
|
|
}
|
|
if (path === "/users/srp/verify-session") {
|
|
const body = JSON.parse(init?.body as string);
|
|
srpServer.checkM1(Buffer.from(body.srpM1, "base64"));
|
|
return json({
|
|
srpM2: srpServer.computeM2().toString("base64"),
|
|
id: 42,
|
|
keyAttributes: m.keyAttributes,
|
|
encryptedToken: m.encryptedToken,
|
|
});
|
|
}
|
|
|
|
// Collections & files
|
|
if (path === "/collections/v2")
|
|
return json({ collections: m.collections });
|
|
if (path === "/collections/v2/diff") {
|
|
const collID = Number(parsed.searchParams.get("collectionID"));
|
|
return json({
|
|
diff: m.filesByCollection[collID] ?? [],
|
|
hasMore: false,
|
|
});
|
|
}
|
|
|
|
// File download (for fix: download original to generate thumb)
|
|
if (
|
|
url.includes("files.ente.io") ||
|
|
path.startsWith("/files/download/")
|
|
) {
|
|
const fileID = Number(
|
|
parsed.searchParams.get("fileID") ?? path.split("/").pop(),
|
|
);
|
|
const ct = m.fileCiphertexts[fileID];
|
|
if (ct) return new Response(ct, { status: 200 });
|
|
return new Response("not found", { status: 404 });
|
|
}
|
|
|
|
// Thumbnail download (for list: check if thumbnail exists)
|
|
if (
|
|
url.includes("thumbnails.ente.io") ||
|
|
path.startsWith("/files/preview/")
|
|
) {
|
|
const fileID = Number(
|
|
parsed.searchParams.get("fileID") ?? path.split("/").pop(),
|
|
);
|
|
const behavior = m.thumbnailBehavior[fileID];
|
|
if (behavior === "ok") {
|
|
return new Response(new Uint8Array([0xff, 0xd8, 0xff]), {
|
|
status: 200,
|
|
});
|
|
}
|
|
if (behavior === "empty") {
|
|
return new Response(new Uint8Array(0), { status: 200 });
|
|
}
|
|
if (behavior === "500") {
|
|
return new Response("Internal Server Error", { status: 500 });
|
|
}
|
|
return new Response("not found", { status: 404 });
|
|
}
|
|
|
|
// Upload URL minting
|
|
if (path === "/files/upload-url") {
|
|
return json({
|
|
objectKey: `42/thumb-${Date.now()}`,
|
|
url: "https://s3.mock.test/presigned-put",
|
|
});
|
|
}
|
|
|
|
// Presigned PUT (S3 upload)
|
|
if (url.startsWith("https://s3.mock.test/")) {
|
|
const body = init?.body;
|
|
// Store the uploaded bytes for later inspection
|
|
if (body instanceof Uint8Array) {
|
|
(m as Record<string, unknown>)._lastUploadedCiphertext = body;
|
|
} else if (body instanceof ArrayBuffer) {
|
|
(m as Record<string, unknown>)._lastUploadedCiphertext =
|
|
new Uint8Array(body);
|
|
}
|
|
return new Response(null, { status: 200 });
|
|
}
|
|
|
|
// Update thumbnail metadata
|
|
if (path === "/files/thumbnail" && init?.method === "PUT") {
|
|
const reqBody = JSON.parse(init?.body as string);
|
|
m.uploadedThumbnails.push({
|
|
fileID: reqBody.fileID,
|
|
objectKey: reqBody.thumbnail.objectKey,
|
|
decryptionHeader: reqBody.thumbnail.decryptionHeader,
|
|
ciphertext:
|
|
((m as Record<string, unknown>)
|
|
._lastUploadedCiphertext as Uint8Array) ??
|
|
new Uint8Array(0),
|
|
});
|
|
return json({});
|
|
}
|
|
|
|
return new Response("not found", { status: 404 });
|
|
}) as typeof globalThis.fetch;
|
|
};
|
|
|
|
/**
|
|
* A retry policy with the waiting removed. `listMissingThumbnails` walks every
|
|
* file in the account, so a transient failure is retried; without an injected
|
|
* `sleep` these tests would spend real seconds waiting out backoff.
|
|
*/
|
|
const noWait: RetryOptions = {
|
|
sleep: () => Promise.resolve(),
|
|
random: () => 0,
|
|
};
|
|
|
|
/** Wrap a fetch so the tests can count how often one endpoint was hit. */
|
|
const countingFetch = (
|
|
inner: typeof globalThis.fetch,
|
|
match: (url: string) => boolean,
|
|
): { fetch: typeof globalThis.fetch; matched: () => number } => {
|
|
let matched = 0;
|
|
const fake = async (
|
|
input: RequestInfo | URL,
|
|
init?: RequestInit,
|
|
): Promise<Response> => {
|
|
const url =
|
|
typeof input === "string"
|
|
? input
|
|
: input instanceof URL
|
|
? input.href
|
|
: input.url;
|
|
if (match(url)) matched++;
|
|
return inner(input, init);
|
|
};
|
|
return { fetch: fake as typeof globalThis.fetch, matched: () => matched };
|
|
};
|
|
|
|
// Open a library over a mock-backed client. As the CLI does for point commands,
|
|
// the background precache is off and the refresh interval is long, and the
|
|
// library client omits `fetchMLData` so no background ML fetch runs. The real
|
|
// `Client` is still used for the API operations the helpers perform directly.
|
|
const openLib = (client: Client): Promise<Library> =>
|
|
Library.open({
|
|
client: {
|
|
whoami: () => client.whoami(),
|
|
collectionsSince: (args) => client.collectionsSince(args),
|
|
filesSince: (args) => client.filesSince(args),
|
|
contentSource: () => client.contentSource(),
|
|
},
|
|
cacheDirectory: mkdtempSync(join(tmpRoot, "cache-")),
|
|
refreshIntervalSeconds: 3600,
|
|
precacheThumbnails: false,
|
|
precacheOriginals: false,
|
|
});
|
|
|
|
/** The mock's raw record for one file, for a test to change before login. */
|
|
const rawFile = (m: ThumbMockState, fileID: number): Record<string, unknown> =>
|
|
m.filesByCollection[1]!.find((f) => f.id === fileID)!;
|
|
|
|
/** Replace the original the mock serves for one file. */
|
|
const replaceOriginal = (
|
|
m: ThumbMockState,
|
|
fileID: number,
|
|
body: Uint8Array,
|
|
): void => {
|
|
const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(
|
|
m.fileKeys[fileID]!,
|
|
);
|
|
m.fileCiphertexts[fileID] =
|
|
sodium.crypto_secretstream_xchacha20poly1305_push(
|
|
push.state,
|
|
body,
|
|
null,
|
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
|
);
|
|
rawFile(m, fileID).file = { decryptionHeader: toBase64(push.header) };
|
|
};
|
|
|
|
const isOriginalDownload = (url: string): boolean =>
|
|
url.includes("files.ente.io") || url.includes("/files/download/");
|
|
|
|
const login = (fetch: typeof globalThis.fetch, retry?: RetryOptions) =>
|
|
Client.login({
|
|
email: TEST_EMAIL,
|
|
password: TEST_PASSWORD,
|
|
apiOptions: retry ? { fetch, retry } : { fetch },
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
beforeAll(async () => {
|
|
await init();
|
|
await sodium.ready;
|
|
mock = await buildThumbMock();
|
|
tmpRoot = mkdtempSync(join(tmpdir(), "quak-thumb-test-"));
|
|
});
|
|
|
|
afterAll(() => {
|
|
if (tmpRoot && existsSync(tmpRoot))
|
|
rmSync(tmpRoot, { recursive: true, force: true });
|
|
});
|
|
|
|
describe("listMissingThumbnails", () => {
|
|
it("identifies files with empty and 404 thumbnails, ignores working ones", async () => {
|
|
const client = await login(buildThumbFetch(mock));
|
|
const lib = await openLib(client);
|
|
|
|
const missing = await listMissingThumbnails(lib, client);
|
|
lib.close();
|
|
|
|
// File 100 has a working thumbnail → not reported
|
|
// File 101 has an empty thumbnail → reported
|
|
// File 102 has a 404 thumbnail → reported
|
|
expect(missing.length).toBe(2);
|
|
const ids = missing.map((m) => m.fileID).sort();
|
|
expect(ids).toEqual([101, 102]);
|
|
|
|
const emptyEntry = missing.find((m) => m.fileID === 101)!;
|
|
expect(emptyEntry.reason).toContain("empty");
|
|
expect(emptyEntry.title).toBe("file-101.jpg");
|
|
expect(emptyEntry.collection).toBe("Photos");
|
|
|
|
const notFoundEntry = missing.find((m) => m.fileID === 102)!;
|
|
// A 404 is the server stating the thumbnail is not there. That is the
|
|
// only network answer that means "missing", and the reason says so
|
|
// rather than the older catch-all "fetch failed" — which used to
|
|
// cover a 500 and a dropped connection too.
|
|
expect(notFoundEntry.reason).toContain("not found");
|
|
});
|
|
|
|
it("does not report a thumbnail as missing when the server is failing", async () => {
|
|
// The distinction that matters for `helper fix-missing-thumbnails`.
|
|
// Reporting a file here leads to downloading the original,
|
|
// regenerating a thumbnail, and uploading it over a thumbnail that
|
|
// was fine all along — because the server was briefly returning 500s.
|
|
//
|
|
// File 102 serves 500 on every attempt, so the retries are genuinely
|
|
// exhausted. It must still not be reported.
|
|
const failingMock = await buildThumbMock();
|
|
failingMock.thumbnailBehavior[102] = "500";
|
|
|
|
const counted = countingFetch(
|
|
buildThumbFetch(failingMock),
|
|
(url) => url.includes("thumbnails.ente.io") && url.includes("102"),
|
|
);
|
|
const client = await login(counted.fetch, { ...noWait });
|
|
const lib = await openLib(client);
|
|
|
|
const missing = await listMissingThumbnails(lib, client);
|
|
lib.close();
|
|
|
|
// Only the genuinely empty thumbnail is reported.
|
|
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
|
// And the 500 was retried rather than accepted as an answer: four
|
|
// attempts is the library default.
|
|
expect(counted.matched()).toBe(4);
|
|
});
|
|
|
|
it("does not report a thumbnail as missing when the connection fails", async () => {
|
|
// Same rule for a transport failure, which carries no status at all.
|
|
const failingMock = await buildThumbMock();
|
|
const inner = buildThumbFetch(failingMock);
|
|
let thumbRequests = 0;
|
|
const fetch = (async (
|
|
input: RequestInfo | URL,
|
|
init?: RequestInit,
|
|
): Promise<Response> => {
|
|
const url =
|
|
typeof input === "string"
|
|
? input
|
|
: input instanceof URL
|
|
? input.href
|
|
: input.url;
|
|
if (url.includes("thumbnails.ente.io") && url.includes("102")) {
|
|
thumbRequests++;
|
|
throw Object.assign(new Error("socket hang up"), {
|
|
code: "ECONNRESET",
|
|
});
|
|
}
|
|
return inner(input, init);
|
|
}) as typeof globalThis.fetch;
|
|
|
|
const client = await login(fetch, { ...noWait });
|
|
const lib = await openLib(client);
|
|
|
|
const missing = await listMissingThumbnails(lib, client);
|
|
lib.close();
|
|
|
|
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
|
expect(thumbRequests).toBe(4);
|
|
});
|
|
|
|
it("deduplicates files seen in multiple collections", async () => {
|
|
// Add the same files to a second collection in the mock
|
|
const mockWithDupes = await buildThumbMock();
|
|
const dupeCollection = {
|
|
...mockWithDupes.collections[0]!,
|
|
id: 2,
|
|
};
|
|
mockWithDupes.collections.push(
|
|
dupeCollection as Record<string, unknown>,
|
|
);
|
|
mockWithDupes.filesByCollection[2] =
|
|
mockWithDupes.filesByCollection[1]!;
|
|
|
|
const client = await login(buildThumbFetch(mockWithDupes));
|
|
const lib = await openLib(client);
|
|
|
|
const missing = await listMissingThumbnails(lib, client);
|
|
lib.close();
|
|
|
|
// Should still be 2, not 4 (each file checked only once)
|
|
expect(missing.length).toBe(2);
|
|
});
|
|
|
|
it("skips a file another account owns without fetching its thumbnail", async () => {
|
|
const otherMock = await buildThumbMock();
|
|
rawFile(otherMock, 102).ownerID = 7;
|
|
const logs: string[] = [];
|
|
const counted = countingFetch(
|
|
buildThumbFetch(otherMock),
|
|
(url) => url.includes("thumbnails.ente.io") && url.includes("102"),
|
|
);
|
|
const client = await login(counted.fetch);
|
|
const lib = await openLib(client);
|
|
|
|
const missing = await listMissingThumbnails(lib, client, (msg) =>
|
|
logs.push(msg),
|
|
);
|
|
lib.close();
|
|
|
|
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
|
expect(counted.matched()).toBe(0);
|
|
expect(
|
|
logs.some(
|
|
(l) =>
|
|
l.includes("Skipping file-102.jpg") &&
|
|
l.includes("another account"),
|
|
),
|
|
).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe("fixMissingThumbnails", () => {
|
|
it("downloads original, generates thumbnail, encrypts, uploads, and registers", async () => {
|
|
const fixMock = await buildThumbMock();
|
|
const client = await login(buildThumbFetch(fixMock));
|
|
const lib = await openLib(client);
|
|
|
|
const results = await fixMissingThumbnails(lib, client, [101]);
|
|
lib.close();
|
|
|
|
expect(results.length).toBe(1);
|
|
expect(results[0]!.status).toBe("fixed");
|
|
expect(results[0]!.fileID).toBe(101);
|
|
expect(results[0]!.title).toBe("file-101.jpg");
|
|
expect(results[0]!.collection).toBe("Photos");
|
|
|
|
// Verify the uploaded thumbnail was registered
|
|
expect(fixMock.uploadedThumbnails.length).toBe(1);
|
|
const upload = fixMock.uploadedThumbnails[0]!;
|
|
expect(upload.fileID).toBe(101);
|
|
expect(upload.objectKey).toContain("42/");
|
|
expect(upload.decryptionHeader.length).toBeGreaterThan(0);
|
|
|
|
// Verify the uploaded ciphertext can be decrypted back to a JPEG
|
|
const fileKey = fixMock.fileKeys[101]!;
|
|
const decrypted = decryptBlob(
|
|
upload.ciphertext,
|
|
fromBase64(upload.decryptionHeader),
|
|
fileKey,
|
|
);
|
|
// JPEG magic bytes: FF D8 FF
|
|
expect(decrypted[0]).toBe(0xff);
|
|
expect(decrypted[1]).toBe(0xd8);
|
|
expect(decrypted[2]).toBe(0xff);
|
|
// Verify jpeg-js produced a reasonably sized thumbnail
|
|
expect(decrypted.length).toBeGreaterThan(100);
|
|
expect(decrypted.length).toBeLessThan(50000);
|
|
});
|
|
|
|
it("reports failure for nonexistent file IDs without crashing", async () => {
|
|
const fixMock = await buildThumbMock();
|
|
const client = await login(buildThumbFetch(fixMock));
|
|
const lib = await openLib(client);
|
|
|
|
const results = await fixMissingThumbnails(lib, client, [999]);
|
|
lib.close();
|
|
|
|
expect(results.length).toBe(1);
|
|
expect(results[0]!.status).toBe("failed");
|
|
expect(results[0]!.fileID).toBe(999);
|
|
expect(results[0]!.reason).toContain("not found");
|
|
});
|
|
|
|
it("continues after one file fails and reports mixed results", async () => {
|
|
const fixMock = await buildThumbMock();
|
|
// Make file 102 fail by removing its ciphertext so the download 404s.
|
|
delete fixMock.fileCiphertexts[102];
|
|
|
|
const client = await login(buildThumbFetch(fixMock));
|
|
const lib = await openLib(client);
|
|
|
|
const results = await fixMissingThumbnails(lib, client, [101, 102]);
|
|
lib.close();
|
|
|
|
expect(results.length).toBe(2);
|
|
const success = results.find((r) => r.fileID === 101)!;
|
|
const failure = results.find((r) => r.fileID === 102)!;
|
|
expect(success.status).toBe("fixed");
|
|
expect(failure.status).toBe("failed");
|
|
});
|
|
|
|
it("skips a non-JPEG image and a video as unsupported, not failed (issue #17)", async () => {
|
|
// A PNG and a video both throw inside the JPEG decoder. The helper must
|
|
// recognise them up front and report "skipped", distinct from a genuine
|
|
// "failed", and must not upload anything for them. The JPEG in the same
|
|
// batch is still repaired.
|
|
const fixMock = await buildThumbMock({ extraFormats: true });
|
|
const client = await login(buildThumbFetch(fixMock));
|
|
const lib = await openLib(client);
|
|
|
|
const results = await fixMissingThumbnails(
|
|
lib,
|
|
client,
|
|
[101, 103, 104],
|
|
);
|
|
lib.close();
|
|
|
|
const jpeg = results.find((r) => r.fileID === 101)!;
|
|
const png = results.find((r) => r.fileID === 103)!;
|
|
const video = results.find((r) => r.fileID === 104)!;
|
|
|
|
expect(jpeg.status).toBe("fixed");
|
|
|
|
// The PNG is a still image but not a JPEG: skipped only after its bytes
|
|
// are inspected.
|
|
expect(png.status).toBe("skipped");
|
|
expect(png.reason).toContain("JPEG");
|
|
|
|
// The video is skipped from its type alone, before any download.
|
|
expect(video.status).toBe("skipped");
|
|
expect(video.reason).toContain("video");
|
|
|
|
// Only the JPEG was uploaded; the two skipped files touched no upload.
|
|
expect(fixMock.uploadedThumbnails.length).toBe(1);
|
|
expect(fixMock.uploadedThumbnails[0]!.fileID).toBe(101);
|
|
});
|
|
|
|
it("skips a file another account owns without downloading it", async () => {
|
|
// The server accepts a thumbnail only from the file's owner.
|
|
const fixMock = await buildThumbMock();
|
|
rawFile(fixMock, 101).ownerID = 7;
|
|
const counted = countingFetch(
|
|
buildThumbFetch(fixMock),
|
|
isOriginalDownload,
|
|
);
|
|
const client = await login(counted.fetch);
|
|
const lib = await openLib(client);
|
|
|
|
const results = await fixMissingThumbnails(lib, client, [101]);
|
|
lib.close();
|
|
|
|
expect(results[0]!.status).toBe("skipped");
|
|
expect(results[0]!.reason).toContain("another account");
|
|
expect(counted.matched()).toBe(0);
|
|
expect(fixMock.uploadedThumbnails.length).toBe(0);
|
|
});
|
|
|
|
it("skips a file whose recorded thumbnail size is 0 without downloading it", async () => {
|
|
// The server refuses a thumbnail larger than the one it records, and
|
|
// no thumbnail is 0 bytes.
|
|
const fixMock = await buildThumbMock();
|
|
rawFile(fixMock, 101).info = { thumbSize: 0 };
|
|
const counted = countingFetch(
|
|
buildThumbFetch(fixMock),
|
|
isOriginalDownload,
|
|
);
|
|
const client = await login(counted.fetch);
|
|
const lib = await openLib(client);
|
|
|
|
const results = await fixMissingThumbnails(lib, client, [101]);
|
|
lib.close();
|
|
|
|
expect(results[0]!.status).toBe("skipped");
|
|
expect(results[0]!.reason).toContain("recorded thumbnail size is 0");
|
|
expect(counted.matched()).toBe(0);
|
|
expect(fixMock.uploadedThumbnails.length).toBe(0);
|
|
});
|
|
|
|
it("re-encodes smaller until the thumbnail fits the recorded size", async () => {
|
|
// A noisy 400x300 JPEG, which the default encoding (quality 50, not
|
|
// resized because it is under 720 px) cannot compress below the size
|
|
// recorded here: one byte less than that encoding's ciphertext.
|
|
const fixMock = await buildThumbMock();
|
|
const w = 400;
|
|
const h = 300;
|
|
const noisy = new Uint8Array(
|
|
jpegJs.encode(
|
|
{
|
|
data: sodium.randombytes_buf(w * h * 4),
|
|
width: w,
|
|
height: h,
|
|
},
|
|
90,
|
|
).data,
|
|
);
|
|
replaceOriginal(fixMock, 101, noisy);
|
|
const decoded = jpegJs.decode(noisy, {
|
|
useTArray: true,
|
|
formatAsRGBA: true,
|
|
});
|
|
const defaultSize =
|
|
jpegJs.encode(decoded, 50).data.length +
|
|
sodium.crypto_secretstream_xchacha20poly1305_ABYTES;
|
|
const recordedSize = defaultSize - 1;
|
|
rawFile(fixMock, 101).info = { thumbSize: recordedSize };
|
|
|
|
const client = await login(buildThumbFetch(fixMock));
|
|
const lib = await openLib(client);
|
|
|
|
const results = await fixMissingThumbnails(lib, client, [101]);
|
|
lib.close();
|
|
|
|
expect(results[0]!.status).toBe("fixed");
|
|
const upload = fixMock.uploadedThumbnails[0]!;
|
|
expect(upload.ciphertext.length).toBeLessThanOrEqual(recordedSize);
|
|
const decrypted = decryptBlob(
|
|
upload.ciphertext,
|
|
fromBase64(upload.decryptionHeader),
|
|
fixMock.fileKeys[101]!,
|
|
);
|
|
expect(decrypted[0]).toBe(0xff);
|
|
expect(decrypted[1]).toBe(0xd8);
|
|
});
|
|
});
|
|
|
|
describe("Client.getApiClient", () => {
|
|
it("returns the ApiClient when logged in", async () => {
|
|
const client = await login(buildThumbFetch(mock));
|
|
|
|
const api = client.getApiClient();
|
|
expect(api).toBeDefined();
|
|
expect(typeof api.getJSON).toBe("function");
|
|
});
|
|
|
|
it("throws after logout", async () => {
|
|
const client = await login(buildThumbFetch(mock));
|
|
client.logout();
|
|
|
|
expect(() => client.getApiClient()).toThrow(/logged out/);
|
|
});
|
|
});
|