Route every command through Library.open instead of scanning the client. The read commands (collections, files, get, get-thumb) force a server round-trip with Library.fresh() before reading, so they answer for current state, not a stale cache (owner amendment, issue #36). collections and files list in the library's enumeration order — the order the pre-library CLI printed, not the newest-first projection — and present each file from its own decrypted metadata (raw title, microsecond creationTime). get/get-thumb copy the cached original/thumbnail to --out. backup, backup-metadata, and the thumbnail helpers are unchanged. A new global --cache-dir sets the cache location; point commands open with the background precache off so a one-shot command never downloads the account. Also addresses #17: fix-missing-thumbnails reports a non-JPEG image or a video as skipped (unsupported), distinct from failed, and only a genuine failure exits non-zero. Model: opus-4-8
This commit is contained in:
@@ -2,13 +2,17 @@
|
||||
* Tests for `quak backup-metadata <dir>`.
|
||||
*
|
||||
* This command dumps all decrypted account metadata into a directory
|
||||
* tree of plain JSON files, without downloading any file content. It
|
||||
* is fast (no multi-megabyte downloads) and produces a complete
|
||||
* plaintext record of every collection name, file title, creation
|
||||
* date, GPS coordinate, camera model, caption, face label, and any
|
||||
* other metadata the Ente clients have attached.
|
||||
* tree of plain JSON files, without downloading any file content (unless
|
||||
* `--exif` is given). It is fast and produces a complete plaintext record of
|
||||
* every collection name, file title, creation date, GPS coordinate, camera
|
||||
* model, caption, face label, and any other metadata the Ente clients have
|
||||
* attached.
|
||||
*
|
||||
* Layout:
|
||||
* As of issue #52 it runs on the library API: `runMetadataBackup(lib, client,
|
||||
* dir)` enumerates collections and files from the library's cache rather than
|
||||
* scanning the client directly, and `--exif` reads each original through the
|
||||
* library's content cache (`photo.original()`). The ML fetch is unchanged. The
|
||||
* output tree is identical:
|
||||
*
|
||||
* <dir>/
|
||||
* account.json { email, userID }
|
||||
@@ -44,7 +48,11 @@ import {
|
||||
} from "../../src/crypto/index.js";
|
||||
import * as jpegJs from "jpeg-js";
|
||||
import { Client } from "../../src/client.js";
|
||||
import { runMetadataBackup } from "../../src/metadata-backup.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
import {
|
||||
runMetadataBackup,
|
||||
type MetadataBackupOptions,
|
||||
} from "../../src/metadata-backup.js";
|
||||
import type { KeyAttributes } from "../../src/auth/types.js";
|
||||
|
||||
const TEST_EMAIL = "metabackup@example.com";
|
||||
@@ -431,16 +439,50 @@ afterAll(() => {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Log in against the mock and open a library over its cache. The point commands
|
||||
// open the library with the background precache off and a long refresh interval;
|
||||
// the same here keeps the test deterministic (no thumbnail/original prefetch it
|
||||
// did not ask for, no second refresh mid-test).
|
||||
const openLib = async (client: Client): Promise<Library> =>
|
||||
Library.open({
|
||||
// The library client omits `fetchMLData`, matching how the CLI opens
|
||||
// point commands: `runMetadataBackup` fetches ML data itself through
|
||||
// the client, so the library's background backfill would only be a
|
||||
// redundant second pass over the same endpoint.
|
||||
client: {
|
||||
whoami: () => client.whoami(),
|
||||
collectionsSince: (args) => client.collectionsSince(args),
|
||||
filesSince: (args) => client.filesSince(args),
|
||||
contentSource: () => client.contentSource(),
|
||||
},
|
||||
cacheDirectory: mkdtempSync(join(testDir, "cache-")),
|
||||
refreshIntervalSeconds: 3600,
|
||||
precacheThumbnails: false,
|
||||
precacheOriginals: false,
|
||||
});
|
||||
|
||||
// Run one metadata backup end to end: fresh client, fresh library, then close.
|
||||
const runBackup = async (
|
||||
outDir: string,
|
||||
opts?: MetadataBackupOptions,
|
||||
): Promise<void> => {
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
const lib = await openLib(client);
|
||||
try {
|
||||
await runMetadataBackup(lib, client, outDir, opts);
|
||||
} finally {
|
||||
lib.close();
|
||||
}
|
||||
};
|
||||
|
||||
describe("quak backup-metadata", () => {
|
||||
it("writes account.json with email and userID", async () => {
|
||||
const outDir = join(testDir, "full");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const account = JSON.parse(
|
||||
readFileSync(join(outDir, "account.json"), "utf-8"),
|
||||
@@ -451,13 +493,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("creates per-collection directories with _collection.json", async () => {
|
||||
const outDir = join(testDir, "collections");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
expect(collDirs.length).toBe(2);
|
||||
@@ -478,13 +514,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("decrypts collection-level pubMagicMetadata", async () => {
|
||||
const outDir = join(testDir, "coll-magic");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
@@ -501,13 +531,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("writes per-file JSON with all three metadata layers", async () => {
|
||||
const outDir = join(testDir, "file-meta");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
@@ -526,13 +550,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("handles files with no magic metadata gracefully", async () => {
|
||||
const outDir = join(testDir, "no-magic");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const workDir = collDirs.find((d) => d.includes("Work"))!;
|
||||
@@ -550,14 +568,8 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("is incremental: second run does not fail", async () => {
|
||||
const outDir = join(testDir, "incremental");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const account = JSON.parse(
|
||||
readFileSync(join(outDir, "account.json"), "utf-8"),
|
||||
@@ -567,13 +579,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("fetches and decrypts ML data by default", async () => {
|
||||
const outDir = join(testDir, "ml-data");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
@@ -597,13 +603,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("extracts EXIF from downloaded files when --exif is set", async () => {
|
||||
const outDir = join(testDir, "exif-data");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir, { exif: true });
|
||||
await runBackup(outDir, { exif: true });
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
// The CLI presents a file by its own decrypted metadata, not the PhotoRecord
|
||||
// projection (issue #52). For a renamed file the two disagree: the projection
|
||||
// prefers `editedName` and reports `editedTime` in milliseconds, while the CLI
|
||||
// must print the raw `metadata.title` and `metadata.creationTime` (microseconds)
|
||||
// and name downloads after the raw title, byte-identical to the pre-library CLI.
|
||||
//
|
||||
// This locks in that contrast: the shared output helpers emit the raw values,
|
||||
// and the projection of the same file emits the edited ones — so a regression
|
||||
// that re-sourced the CLI from the projection would fail here.
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import {
|
||||
fileListRow,
|
||||
fileListLine,
|
||||
originalName,
|
||||
thumbnailName,
|
||||
} from "../../src/cli-output.js";
|
||||
import { deriveRecords } from "../../src/library/records.js";
|
||||
import type { EnteFile } from "../../src/model/types.js";
|
||||
|
||||
// Microseconds, as Ente stores times.
|
||||
const RAW_CREATION = 1700000000000000;
|
||||
const EDITED_TIME = 1710000000000000;
|
||||
const RAW_TITLE = "IMG_0001.HEIC";
|
||||
const EDITED_NAME = "Sunset.heic";
|
||||
|
||||
// A file the user has renamed and re-dated: basic metadata holds the original
|
||||
// title and capture time; public magic metadata holds the edits.
|
||||
const renamedFile: EnteFile = {
|
||||
id: 100,
|
||||
collectionID: 10,
|
||||
ownerID: 42,
|
||||
key: new Uint8Array(),
|
||||
metadata: {
|
||||
title: RAW_TITLE,
|
||||
fileType: "image",
|
||||
creationTime: RAW_CREATION,
|
||||
modificationTime: RAW_CREATION,
|
||||
},
|
||||
pubMagicMetadata: { editedName: EDITED_NAME, editedTime: EDITED_TIME },
|
||||
file: { decryptionHeader: "" },
|
||||
thumbnail: { decryptionHeader: "" },
|
||||
updationTime: RAW_CREATION,
|
||||
};
|
||||
|
||||
describe("CLI file output (issue #52)", () => {
|
||||
it("emits the raw title and microsecond creationTime for --json", () => {
|
||||
expect(fileListRow(renamedFile)).toEqual({
|
||||
id: 100,
|
||||
title: RAW_TITLE,
|
||||
fileType: "image",
|
||||
creationTime: RAW_CREATION,
|
||||
collectionID: 10,
|
||||
});
|
||||
});
|
||||
|
||||
it("emits the raw title in the human column", () => {
|
||||
expect(fileListLine(renamedFile)).toBe(`100\timage\t${RAW_TITLE}`);
|
||||
});
|
||||
|
||||
it("names downloads after the raw title", () => {
|
||||
expect(originalName(renamedFile)).toBe(RAW_TITLE);
|
||||
expect(thumbnailName(renamedFile)).toBe(`thumb_${RAW_TITLE}`);
|
||||
});
|
||||
|
||||
it("does not use the editedName/editedTime projection", () => {
|
||||
const record = deriveRecords([], [renamedFile]).photos.get(100);
|
||||
// The projection prefers the edits and reports milliseconds; the CLI
|
||||
// helpers above deliberately do not.
|
||||
expect(record?.title).toBe(EDITED_NAME);
|
||||
expect(record?.takenAt).toBe(Math.floor(EDITED_TIME / 1000));
|
||||
expect(fileListRow(renamedFile).title).not.toBe(record?.title);
|
||||
expect(fileListRow(renamedFile).creationTime).not.toBe(record?.takenAt);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,195 @@
|
||||
/**
|
||||
* Tests for the CLI read helpers (`src/cli-read.ts`, owner amendment to
|
||||
* issue #36, issue #52).
|
||||
*
|
||||
* The `collections`, `files`, `get`, and `get-thumb` commands must answer for
|
||||
* current server state, not the local cache, so each helper forces a
|
||||
* `Library.fresh()` round-trip before it reads. The stand-in library below
|
||||
* serves nothing until `fresh()` has been awaited, so a helper that read
|
||||
* without refreshing would come back empty and fail here.
|
||||
*
|
||||
* `collections` and `files` also list in the library's enumeration order
|
||||
* (`listCollections`/`listFiles`) — the order the pre-library CLI printed — not
|
||||
* the albums/photos projection's newest-first order. The fixtures are seeded in
|
||||
* an enumeration order that a newest-first sort would rearrange, so a
|
||||
* regression to the projection order would fail here too. Field values still
|
||||
* come from the raw metadata via `cli-output.ts`.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import {
|
||||
freshCollections,
|
||||
freshFiles,
|
||||
freshFile,
|
||||
type FreshReadLibrary,
|
||||
} from "../../src/cli-read.js";
|
||||
import { fileListRow } from "../../src/cli-output.js";
|
||||
import type { Photo } from "../../src/library/index.js";
|
||||
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||
|
||||
const collection = (id: number, updationTime: number): Collection => ({
|
||||
id,
|
||||
ownerID: 42,
|
||||
key: new Uint8Array(),
|
||||
name: `album-${id}`,
|
||||
type: "album",
|
||||
updationTime,
|
||||
isShared: false,
|
||||
});
|
||||
|
||||
// Microseconds, as Ente stores times.
|
||||
const file = (
|
||||
id: number,
|
||||
collectionID: number,
|
||||
creationTime: number,
|
||||
): EnteFile => ({
|
||||
id,
|
||||
collectionID,
|
||||
ownerID: 42,
|
||||
key: new Uint8Array(),
|
||||
metadata: {
|
||||
title: `file-${id}.jpg`,
|
||||
fileType: "image",
|
||||
creationTime,
|
||||
modificationTime: creationTime,
|
||||
},
|
||||
file: { decryptionHeader: "" },
|
||||
thumbnail: { decryptionHeader: "" },
|
||||
updationTime: creationTime,
|
||||
});
|
||||
|
||||
// A library that reveals its records only after `fresh()` has been awaited, and
|
||||
// serves them in the enumeration order it was given. `photos.byID` returns a
|
||||
// stand-in `Photo` carrying just the fileID the helper passes through.
|
||||
class FakeLibrary implements FreshReadLibrary {
|
||||
freshCalls = 0;
|
||||
private refreshed = false;
|
||||
|
||||
constructor(
|
||||
private readonly collections: Collection[],
|
||||
private readonly files: EnteFile[],
|
||||
) {}
|
||||
|
||||
async fresh(): Promise<unknown> {
|
||||
this.freshCalls++;
|
||||
this.refreshed = true;
|
||||
return {};
|
||||
}
|
||||
|
||||
listCollections(): Collection[] {
|
||||
return this.refreshed ? this.collections : [];
|
||||
}
|
||||
|
||||
getCollection(id: number): Collection | undefined {
|
||||
return this.listCollections().find((c) => c.id === id);
|
||||
}
|
||||
|
||||
listFiles(collectionID: number): EnteFile[] {
|
||||
return this.refreshed
|
||||
? this.files.filter((f) => f.collectionID === collectionID)
|
||||
: [];
|
||||
}
|
||||
|
||||
getFileByID(fileID: number): EnteFile | undefined {
|
||||
if (!this.refreshed) return undefined;
|
||||
return this.files.find((f) => f.id === fileID);
|
||||
}
|
||||
|
||||
photos = {
|
||||
byID: ({ fileID }: { fileID: number }): Photo | undefined => {
|
||||
if (!this.refreshed) return undefined;
|
||||
if (!this.files.some((f) => f.id === fileID)) return undefined;
|
||||
return { fileID } as unknown as Photo;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("CLI read helpers (issue #36 amendment, issue #52)", () => {
|
||||
it("freshCollections refreshes first, then lists in enumeration order", async () => {
|
||||
// Enumeration order 2, 1, 3; a newest-first sort would be 3, 2, 1.
|
||||
const lib = new FakeLibrary(
|
||||
[collection(2, 200), collection(1, 300), collection(3, 100)],
|
||||
[],
|
||||
);
|
||||
const rows = await freshCollections(lib);
|
||||
|
||||
expect(lib.freshCalls).toBe(1);
|
||||
expect(rows.map((c) => c.id)).toEqual([2, 1, 3]);
|
||||
// The projection's newest-first order is a different sequence, so this
|
||||
// is not accidentally that order.
|
||||
const newestFirst = [...rows]
|
||||
.sort((a, b) => b.updationTime - a.updationTime)
|
||||
.map((c) => c.id);
|
||||
expect(newestFirst).toEqual([1, 2, 3]);
|
||||
expect(rows.map((c) => c.id)).not.toEqual(newestFirst);
|
||||
});
|
||||
|
||||
it("freshFiles refreshes first, lists in enumeration order, keeps raw fields", async () => {
|
||||
// Enumeration order by id 10, 11, 12; creationTimes ascending, so a
|
||||
// newest-first sort would reverse them.
|
||||
const files = [
|
||||
file(10, 1, 1_700_000_000_000_000),
|
||||
file(11, 1, 1_700_000_000_000_001),
|
||||
file(12, 1, 1_700_000_000_000_002),
|
||||
];
|
||||
const lib = new FakeLibrary([collection(1, 100)], files);
|
||||
|
||||
const rows = await freshFiles(lib, 1);
|
||||
expect(lib.freshCalls).toBe(1);
|
||||
expect(rows?.map((f) => f.id)).toEqual([10, 11, 12]);
|
||||
|
||||
// Field values come from raw metadata: microsecond creationTime and the
|
||||
// raw title, unchanged.
|
||||
expect(rows?.map(fileListRow)).toEqual([
|
||||
{
|
||||
id: 10,
|
||||
title: "file-10.jpg",
|
||||
fileType: "image",
|
||||
creationTime: 1_700_000_000_000_000,
|
||||
collectionID: 1,
|
||||
},
|
||||
{
|
||||
id: 11,
|
||||
title: "file-11.jpg",
|
||||
fileType: "image",
|
||||
creationTime: 1_700_000_000_000_001,
|
||||
collectionID: 1,
|
||||
},
|
||||
{
|
||||
id: 12,
|
||||
title: "file-12.jpg",
|
||||
fileType: "image",
|
||||
creationTime: 1_700_000_000_000_002,
|
||||
collectionID: 1,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("freshFiles returns undefined for an unknown collection", async () => {
|
||||
const lib = new FakeLibrary([collection(1, 100)], []);
|
||||
const rows = await freshFiles(lib, 999);
|
||||
expect(lib.freshCalls).toBe(1);
|
||||
expect(rows).toBeUndefined();
|
||||
});
|
||||
|
||||
it("freshFile refreshes first, then resolves the photo and its raw record", async () => {
|
||||
const f = file(10, 1, 1_700_000_000_000_000);
|
||||
const lib = new FakeLibrary([collection(1, 100)], [f]);
|
||||
|
||||
const resolved = await freshFile(lib, 10);
|
||||
expect(lib.freshCalls).toBe(1);
|
||||
expect(resolved?.photo.fileID).toBe(10);
|
||||
expect(resolved?.file.metadata.title).toBe("file-10.jpg");
|
||||
expect(resolved?.file.metadata.creationTime).toBe(
|
||||
1_700_000_000_000_000,
|
||||
);
|
||||
});
|
||||
|
||||
it("freshFile returns undefined for an unknown file", async () => {
|
||||
const lib = new FakeLibrary([collection(1, 100)], []);
|
||||
const resolved = await freshFile(lib, 404);
|
||||
expect(lib.freshCalls).toBe(1);
|
||||
expect(resolved).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -6,17 +6,30 @@
|
||||
* working thumbnails, others return 404 or empty bodies. The tests
|
||||
* verify that the detection and repair logic handles each case correctly.
|
||||
*
|
||||
* As of issue #52 both helpers take an open `Library` for enumeration and the
|
||||
* `Client` for the API operations that stay unchanged (the thumbnail existence
|
||||
* check, and the encrypt-and-upload path). `fixMissingThumbnails` reads each
|
||||
* original through the library's content cache (`photo.original()`).
|
||||
*
|
||||
* `fixMissingThumbnails` is the most complex function in quak: it
|
||||
* downloads the original file, generates a JPEG thumbnail with jpeg-js,
|
||||
* encrypts it with secretstream push, gets a presigned upload URL,
|
||||
* uploads to S3, and registers the new thumbnail with the API. The
|
||||
* test verifies each step actually happened and the uploaded data is
|
||||
* a valid encrypted blob that decrypts to a JPEG.
|
||||
*
|
||||
* It regenerates thumbnails for baseline JPEGs only, because `jpeg-js` decodes
|
||||
* only JPEG. A non-JPEG image (PNG, HEIC) or a video is reported as "skipped
|
||||
* (unsupported)" rather than crashing the decoder into an opaque failure
|
||||
* (issue #17); the mixed test below locks that distinction down.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import sodium from "libsodium-wrappers-sumo";
|
||||
import * as jpegJs from "jpeg-js";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
init,
|
||||
toBase64,
|
||||
@@ -27,6 +40,7 @@ import {
|
||||
} from "../../src/crypto/index.js";
|
||||
import { SRP, SrpServer } from "fast-srp-hap";
|
||||
import { Client } from "../../src/client.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
import {
|
||||
listMissingThumbnails,
|
||||
fixMissingThumbnails,
|
||||
@@ -42,6 +56,7 @@ const TEST_EMAIL = "thumb@example.com";
|
||||
const TEST_PASSWORD = "thumbpass";
|
||||
const TEST_OPS = 2;
|
||||
const TEST_MEM = 64 * 1024 * 1024;
|
||||
const TEST_TIME = 1700000000000000;
|
||||
|
||||
interface ThumbMockState {
|
||||
verifier: Buffer;
|
||||
@@ -63,8 +78,17 @@ interface ThumbMockState {
|
||||
}
|
||||
|
||||
let mock: ThumbMockState;
|
||||
let tmpRoot: string;
|
||||
|
||||
const buildThumbMock = async (): Promise<ThumbMockState> => {
|
||||
// PNG signature bytes — enough for `fixMissingThumbnails` to recognise a
|
||||
// non-JPEG image and skip it. It need not be a decodable PNG.
|
||||
const PNG_BYTES = new Uint8Array([
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
|
||||
]);
|
||||
|
||||
const buildThumbMock = async (opts?: {
|
||||
extraFormats?: boolean;
|
||||
}): Promise<ThumbMockState> => {
|
||||
const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
|
||||
const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM);
|
||||
const loginSubKeyBytes = deriveLoginSubkey(kek);
|
||||
@@ -102,7 +126,6 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
||||
opsLimit: TEST_OPS,
|
||||
};
|
||||
|
||||
// One collection with 3 files: ok thumbnail, empty thumbnail, 404 thumbnail
|
||||
const collKey = sodium.crypto_secretbox_keygen();
|
||||
const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
||||
const encCK = sodium.crypto_secretbox_easy(collKey, ckN, masterKey);
|
||||
@@ -118,10 +141,11 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
||||
encryptedName: toBase64(encCN),
|
||||
nameDecryptionNonce: toBase64(cnN),
|
||||
type: "album",
|
||||
updationTime: 1700000000000000,
|
||||
updationTime: TEST_TIME,
|
||||
};
|
||||
|
||||
// Generate a real tiny JPEG via jpeg-js
|
||||
// Generate a real tiny JPEG via jpeg-js, used as the encrypted body of the
|
||||
// JPEG files so a repair actually decodes and re-encodes real pixels.
|
||||
const w = 100;
|
||||
const h = 80;
|
||||
const pixels = new Uint8Array(w * h * 4);
|
||||
@@ -131,26 +155,32 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
||||
pixels[i + 2] = 0; // B
|
||||
pixels[i + 3] = 255; // A
|
||||
}
|
||||
const tinyJpeg = jpegJs.encode(
|
||||
{ data: pixels, width: w, height: h },
|
||||
80,
|
||||
).data;
|
||||
const tinyJpeg = new Uint8Array(
|
||||
jpegJs.encode({ data: pixels, width: w, height: h }, 80).data,
|
||||
);
|
||||
|
||||
const fileKeys: Record<number, Uint8Array> = {};
|
||||
const fileCiphertexts: Record<number, Uint8Array> = {};
|
||||
const rawFiles: Record<string, unknown>[] = [];
|
||||
|
||||
for (const fileID of [100, 101, 102]) {
|
||||
// Build one raw file record: encrypt its metadata and its body under a
|
||||
// fresh per-file key, and record the key and ciphertext for the mock to
|
||||
// serve and for the test to verify against.
|
||||
const makeRawFile = (
|
||||
fileID: number,
|
||||
fileType: number,
|
||||
title: string,
|
||||
body: Uint8Array,
|
||||
): Record<string, unknown> => {
|
||||
const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
fileKeys[fileID] = fk;
|
||||
const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
||||
const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey);
|
||||
|
||||
const meta = JSON.stringify({
|
||||
title: `file-${fileID}.jpg`,
|
||||
fileType: 0,
|
||||
creationTime: 1700000000000000,
|
||||
modificationTime: 1700000000000000,
|
||||
title,
|
||||
fileType,
|
||||
creationTime: TEST_TIME,
|
||||
modificationTime: TEST_TIME,
|
||||
});
|
||||
const metaPush =
|
||||
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
||||
@@ -161,18 +191,17 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
||||
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
||||
);
|
||||
|
||||
// Encrypt the tiny JPEG as the file body
|
||||
const filePush =
|
||||
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
||||
const encFile = sodium.crypto_secretstream_xchacha20poly1305_push(
|
||||
filePush.state,
|
||||
new Uint8Array(tinyJpeg),
|
||||
body,
|
||||
null,
|
||||
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
||||
);
|
||||
fileCiphertexts[fileID] = encFile;
|
||||
|
||||
rawFiles.push({
|
||||
return {
|
||||
id: fileID,
|
||||
collectionID: 1,
|
||||
ownerID: 42,
|
||||
@@ -186,8 +215,30 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
||||
thumbnail: {
|
||||
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
||||
},
|
||||
updationTime: 1700000000000000,
|
||||
});
|
||||
updationTime: TEST_TIME,
|
||||
};
|
||||
};
|
||||
|
||||
// Three JPEG files: ok thumbnail, empty thumbnail, 404 thumbnail.
|
||||
const rawFiles: Record<string, unknown>[] = [];
|
||||
for (const fileID of [100, 101, 102]) {
|
||||
rawFiles.push(makeRawFile(fileID, 0, `file-${fileID}.jpg`, tinyJpeg));
|
||||
}
|
||||
const thumbnailBehavior: Record<number, "ok" | "empty" | "404" | "500"> = {
|
||||
100: "ok",
|
||||
101: "empty",
|
||||
102: "404",
|
||||
};
|
||||
|
||||
// For the issue #17 mixed test: a non-JPEG image and a video, both with a
|
||||
// missing (404) thumbnail so they surface in the missing list too.
|
||||
if (opts?.extraFormats) {
|
||||
rawFiles.push(makeRawFile(103, 0, "file-103.png", PNG_BYTES));
|
||||
rawFiles.push(
|
||||
makeRawFile(104, 1, "file-104.mp4", new Uint8Array([0, 0, 0, 1])),
|
||||
);
|
||||
thumbnailBehavior[103] = "404";
|
||||
thumbnailBehavior[104] = "404";
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -206,11 +257,7 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
||||
filesByCollection: { 1: rawFiles },
|
||||
fileCiphertexts,
|
||||
fileKeys,
|
||||
thumbnailBehavior: {
|
||||
100: "ok",
|
||||
101: "empty",
|
||||
102: "404",
|
||||
},
|
||||
thumbnailBehavior,
|
||||
uploadedThumbnails: [],
|
||||
};
|
||||
};
|
||||
@@ -381,6 +428,31 @@ const countingFetch = (
|
||||
return { fetch: fake as typeof globalThis.fetch, matched: () => matched };
|
||||
};
|
||||
|
||||
// Open a library over a mock-backed client. As the CLI does for point commands,
|
||||
// the background precache is off and the refresh interval is long, and the
|
||||
// library client omits `fetchMLData` so no background ML fetch runs. The real
|
||||
// `Client` is still used for the API operations the helpers perform directly.
|
||||
const openLib = (client: Client): Promise<Library> =>
|
||||
Library.open({
|
||||
client: {
|
||||
whoami: () => client.whoami(),
|
||||
collectionsSince: (args) => client.collectionsSince(args),
|
||||
filesSince: (args) => client.filesSince(args),
|
||||
contentSource: () => client.contentSource(),
|
||||
},
|
||||
cacheDirectory: mkdtempSync(join(tmpRoot, "cache-")),
|
||||
refreshIntervalSeconds: 3600,
|
||||
precacheThumbnails: false,
|
||||
precacheOriginals: false,
|
||||
});
|
||||
|
||||
const login = (fetch: typeof globalThis.fetch, retry?: RetryOptions) =>
|
||||
Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: retry ? { fetch, retry } : { fetch },
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -389,17 +461,21 @@ beforeAll(async () => {
|
||||
await init();
|
||||
await sodium.ready;
|
||||
mock = await buildThumbMock();
|
||||
tmpRoot = mkdtempSync(join(tmpdir(), "quak-thumb-test-"));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
if (tmpRoot && existsSync(tmpRoot))
|
||||
rmSync(tmpRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("listMissingThumbnails", () => {
|
||||
it("identifies files with empty and 404 thumbnails, ignores working ones", async () => {
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildThumbFetch(mock) },
|
||||
});
|
||||
const client = await login(buildThumbFetch(mock));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const missing = await listMissingThumbnails(client);
|
||||
const missing = await listMissingThumbnails(lib, client);
|
||||
lib.close();
|
||||
|
||||
// File 100 has a working thumbnail → not reported
|
||||
// File 101 has an empty thumbnail → reported
|
||||
@@ -436,13 +512,11 @@ describe("listMissingThumbnails", () => {
|
||||
buildThumbFetch(failingMock),
|
||||
(url) => url.includes("thumbnails.ente.io") && url.includes("102"),
|
||||
);
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: counted.fetch, retry: { ...noWait } },
|
||||
});
|
||||
const client = await login(counted.fetch, { ...noWait });
|
||||
const lib = await openLib(client);
|
||||
|
||||
const missing = await listMissingThumbnails(client);
|
||||
const missing = await listMissingThumbnails(lib, client);
|
||||
lib.close();
|
||||
|
||||
// Only the genuinely empty thumbnail is reported.
|
||||
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
||||
@@ -475,13 +549,11 @@ describe("listMissingThumbnails", () => {
|
||||
return inner(input, init);
|
||||
}) as typeof globalThis.fetch;
|
||||
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch, retry: { ...noWait } },
|
||||
});
|
||||
const client = await login(fetch, { ...noWait });
|
||||
const lib = await openLib(client);
|
||||
|
||||
const missing = await listMissingThumbnails(client);
|
||||
const missing = await listMissingThumbnails(lib, client);
|
||||
lib.close();
|
||||
|
||||
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
||||
expect(thumbRequests).toBe(4);
|
||||
@@ -500,13 +572,11 @@ describe("listMissingThumbnails", () => {
|
||||
mockWithDupes.filesByCollection[2] =
|
||||
mockWithDupes.filesByCollection[1]!;
|
||||
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildThumbFetch(mockWithDupes) },
|
||||
});
|
||||
const client = await login(buildThumbFetch(mockWithDupes));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const missing = await listMissingThumbnails(client);
|
||||
const missing = await listMissingThumbnails(lib, client);
|
||||
lib.close();
|
||||
|
||||
// Should still be 2, not 4 (each file checked only once)
|
||||
expect(missing.length).toBe(2);
|
||||
@@ -516,16 +586,14 @@ describe("listMissingThumbnails", () => {
|
||||
describe("fixMissingThumbnails", () => {
|
||||
it("downloads original, generates thumbnail, encrypts, uploads, and registers", async () => {
|
||||
const fixMock = await buildThumbMock();
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildThumbFetch(fixMock) },
|
||||
});
|
||||
const client = await login(buildThumbFetch(fixMock));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(client, [101]);
|
||||
const results = await fixMissingThumbnails(lib, client, [101]);
|
||||
lib.close();
|
||||
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0]!.success).toBe(true);
|
||||
expect(results[0]!.status).toBe("fixed");
|
||||
expect(results[0]!.fileID).toBe(101);
|
||||
expect(results[0]!.title).toBe("file-101.jpg");
|
||||
expect(results[0]!.collection).toBe("Photos");
|
||||
@@ -555,48 +623,76 @@ describe("fixMissingThumbnails", () => {
|
||||
|
||||
it("reports failure for nonexistent file IDs without crashing", async () => {
|
||||
const fixMock = await buildThumbMock();
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildThumbFetch(fixMock) },
|
||||
});
|
||||
const client = await login(buildThumbFetch(fixMock));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(client, [999]);
|
||||
const results = await fixMissingThumbnails(lib, client, [999]);
|
||||
lib.close();
|
||||
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0]!.success).toBe(false);
|
||||
expect(results[0]!.status).toBe("failed");
|
||||
expect(results[0]!.fileID).toBe(999);
|
||||
expect(results[0]!.error).toContain("not found");
|
||||
expect(results[0]!.reason).toContain("not found");
|
||||
});
|
||||
|
||||
it("continues after one file fails and reports mixed results", async () => {
|
||||
const fixMock = await buildThumbMock();
|
||||
// Make file 102 fail by removing its ciphertext so download fails
|
||||
// Make file 102 fail by removing its ciphertext so the download 404s.
|
||||
delete fixMock.fileCiphertexts[102];
|
||||
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildThumbFetch(fixMock) },
|
||||
});
|
||||
const client = await login(buildThumbFetch(fixMock));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(client, [101, 102]);
|
||||
const results = await fixMissingThumbnails(lib, client, [101, 102]);
|
||||
lib.close();
|
||||
|
||||
expect(results.length).toBe(2);
|
||||
const success = results.find((r) => r.fileID === 101)!;
|
||||
const failure = results.find((r) => r.fileID === 102)!;
|
||||
expect(success.success).toBe(true);
|
||||
expect(failure.success).toBe(false);
|
||||
expect(success.status).toBe("fixed");
|
||||
expect(failure.status).toBe("failed");
|
||||
});
|
||||
|
||||
it("skips a non-JPEG image and a video as unsupported, not failed (issue #17)", async () => {
|
||||
// A PNG and a video both throw inside the JPEG decoder. The helper must
|
||||
// recognise them up front and report "skipped", distinct from a genuine
|
||||
// "failed", and must not upload anything for them. The JPEG in the same
|
||||
// batch is still repaired.
|
||||
const fixMock = await buildThumbMock({ extraFormats: true });
|
||||
const client = await login(buildThumbFetch(fixMock));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
[101, 103, 104],
|
||||
);
|
||||
lib.close();
|
||||
|
||||
const jpeg = results.find((r) => r.fileID === 101)!;
|
||||
const png = results.find((r) => r.fileID === 103)!;
|
||||
const video = results.find((r) => r.fileID === 104)!;
|
||||
|
||||
expect(jpeg.status).toBe("fixed");
|
||||
|
||||
// The PNG is a still image but not a JPEG: skipped only after its bytes
|
||||
// are inspected.
|
||||
expect(png.status).toBe("skipped");
|
||||
expect(png.reason).toContain("JPEG");
|
||||
|
||||
// The video is skipped from its type alone, before any download.
|
||||
expect(video.status).toBe("skipped");
|
||||
expect(video.reason).toContain("video");
|
||||
|
||||
// Only the JPEG was uploaded; the two skipped files touched no upload.
|
||||
expect(fixMock.uploadedThumbnails.length).toBe(1);
|
||||
expect(fixMock.uploadedThumbnails[0]!.fileID).toBe(101);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Client.getApiClient", () => {
|
||||
it("returns the ApiClient when logged in", async () => {
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildThumbFetch(mock) },
|
||||
});
|
||||
const client = await login(buildThumbFetch(mock));
|
||||
|
||||
const api = client.getApiClient();
|
||||
expect(api).toBeDefined();
|
||||
@@ -604,11 +700,7 @@ describe("Client.getApiClient", () => {
|
||||
});
|
||||
|
||||
it("throws after logout", async () => {
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildThumbFetch(mock) },
|
||||
});
|
||||
const client = await login(buildThumbFetch(mock));
|
||||
client.logout();
|
||||
|
||||
expect(() => client.getApiClient()).toThrow(/logged out/);
|
||||
|
||||
Reference in New Issue
Block a user