sneak/quak
1
0
Fork 0
Code Issues 8 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Port the quak CLI to the library API (closes #52)
check / check (push) Successful in 17s
Details

Browse Source 
Ports the CLI to the library API. collections/files/get/get-thumb use the fresh read variants (Library.fresh(), current server state); backup runs lib.backup; backup-metadata and the missing-thumbnail helpers enumerate via the library. files/get output is byte-identical to the pre-port CLI — raw metadata.title, microsecond creationTime, pre-port row order — exit codes unchanged. Adds --cache-dir; fixes the helper JPEG-only assumption (closes #17).

Model: opus-4-8
This commit was merged in pull request #74.
This commit is contained in:
clawbot
2026-09-23 00:00:55 +02:00
parent aeccb489b5
commit d23d3f8f47
11 changed files with 976 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+15 -3
View File
@@ -420,6 +420,7 @@ you would treat the password itself.
### CLI surface ### CLI surface
``` ```
quak [--cache-dir <path>] <command> global: local metadata/content cache location
quak login interactive or QUAK_EMAIL/QUAK_PASSWORD quak login interactive or QUAK_EMAIL/QUAK_PASSWORD
quak whoami print logged-in account as JSON quak whoami print logged-in account as JSON
quak logout delete saved session quak logout delete saved session
@@ -432,9 +433,20 @@ quak helper list-missing-thumbnails [--json] find files with missing thumbnai
quak helper fix-missing-thumbnails [--file ids] generate + upload missing thumbnails quak helper fix-missing-thumbnails [--file ids] generate + upload missing thumbnails
``` ```
`get` and `get-thumb` search all collections for the file ID when `--collection` Every command reads through the local library cache: the first run fetches the
is not specified. All listing and backup commands support `--json` for account's metadata from the server, and later runs serve from the cache and
machine-readable output. refresh in the background. `--cache-dir` overrides where that cache lives;
without it each account gets its own directory under the per-user cache path.
`get` and `get-thumb` resolve the file by ID directly, so `--collection` is
accepted for backward compatibility but ignored. All listing and backup commands
support `--json` for machine-readable output.
`helper fix-missing-thumbnails` regenerates thumbnails for baseline JPEG images
only, because the bundled decoder (`jpeg-js`) decodes only JPEG. A non-JPEG
image (PNG, HEIC) or a video is reported as `skipped` (unsupported format), kept
distinct from a `failed` repair, and does not affect the exit code; a genuine
failure still exits non-zero.
### Backup layout ### Backup layout
bin/quak.ts
+168 -107
View File
@@ -2,13 +2,26 @@
import { input, password as passwordPrompt } from "@inquirer/prompts"; import { input, password as passwordPrompt } from "@inquirer/prompts";
import { stdout, stderr } from "node:process"; import { stdout, stderr } from "node:process";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import {
copyFileSync,
existsSync,
mkdirSync,
readFileSync,
writeFileSync,
} from "node:fs";
import { join } from "node:path"; import { join } from "node:path";
import { Command } from "commander"; import { Command } from "commander";
import envPaths from "env-paths"; import envPaths from "env-paths";
import { Client, type ClientSnapshot } from "../src/client.js"; import { Client, type ClientSnapshot } from "../src/client.js";
import { init } from "../src/crypto/index.js"; import { init } from "../src/crypto/index.js";
import { Library } from "../src/library/index.js"; import { Library, type LibraryClient } from "../src/library/index.js";
import {
fileListRow,
fileListLine,
originalName,
thumbnailName,
} from "../src/cli-output.js";
import { freshCollections, freshFiles, freshFile } from "../src/cli-read.js";
import { runMetadataBackup } from "../src/metadata-backup.js"; import { runMetadataBackup } from "../src/metadata-backup.js";
import { import {
listMissingThumbnails, listMissingThumbnails,
@@ -55,7 +68,61 @@ const program = new Command();
program program
.name("quak") .name("quak")
.description("CLI for the Ente end-to-end encrypted photo service") .description("CLI for the Ente end-to-end encrypted photo service")
.version("0.0.0"); .version("0.0.0")
.option(
"--cache-dir <path>",
"Directory for the local metadata/content cache " +
"(default: the per-user cache directory)",
);
// The `--cache-dir` global, or undefined to let the library pick its per-user
// default keyed by the account id.
const cacheDirOption = (): string | undefined =>
program.opts<{ cacheDir?: string }>().cacheDir;
// A library client that omits `fetchMLData`, so the point commands below do not
// kick the library's background ML backfill: they read metadata, or fetch one
// file's content, and exit. `backup` and `backup-metadata` handle ML on their
// own terms. The content source is kept so `get`/`get-thumb`/`--exif` can fetch
// originals through the on-disk cache.
const readLibraryClient = (client: Client): LibraryClient => ({
whoami: () => client.whoami(),
collectionsSince: (args) => client.collectionsSince(args),
filesSince: (args) => client.filesSince(args),
contentSource: () => client.contentSource(),
});
// Open a library for a single point command: the aggressive background precache
// (issue #48) is off — a one-shot `collections` or `get` must not start
// downloading the whole account — and the refresh interval is long so no second
// refresh fires mid-command.
const openReadLibrary = (client: Client): Promise<Library> =>
Library.open({
client: readLibraryClient(client),
cacheDirectory: cacheDirOption(),
refreshIntervalSeconds: 3600,
precacheThumbnails: false,
precacheOriginals: false,
});
// Close the library and exit once stdout/stderr have drained. `process.exit`
// alone can truncate buffered piped output, and the library keeps the event
// loop alive with a background refresh, so a plain return could hang; this does
// neither.
const finish = (lib: Library | undefined, code: number): void => {
lib?.close();
const pending = [stdout, stderr].filter((s) => s.writableLength > 0);
if (pending.length === 0) {
process.exit(code);
return;
}
let remaining = pending.length;
for (const s of pending) {
s.once("drain", () => {
if (--remaining === 0) process.exit(code);
});
}
};
program program
.command("login") .command("login")
@@ -116,7 +183,11 @@ program
.action(async (opts: { json?: boolean }) => { .action(async (opts: { json?: boolean }) => {
await init(); await init();
const client = requireSession(); const client = requireSession();
const collections = await client.listCollections(); const lib = await openReadLibrary(client);
// Force a server round-trip and list in enumeration order (issue #36
// amendment, issue #52): the pre-library CLI printed current state in
// this order, not the albums projection's newest-first order.
const collections = await freshCollections(lib);
if (opts.json) { if (opts.json) {
stdout.write( stdout.write(
@@ -140,6 +211,7 @@ program
); );
} }
} }
finish(lib, 0);
}); });
program program
@@ -159,36 +231,29 @@ program
process.exit(1); process.exit(1);
} }
const collections = await client.listCollections(); const lib = await openReadLibrary(client);
const col = collections.find((c) => c.id === collectionID); // Force a server round-trip and list in enumeration order (issue #36
if (!col) { // amendment, issue #52). Each file prints from its own decrypted
// metadata (raw title, microsecond creationTime) via cli-output, and in
// the pre-library CLI's enumeration order, not the projection's
// newest-first order.
const files = await freshFiles(lib, collectionID);
if (!files) {
stderr.write(`Collection ${collectionID} not found\n`); stderr.write(`Collection ${collectionID} not found\n`);
process.exit(1); finish(lib, 1);
return;
} }
const files = await client.listFiles(col.id, col.key);
if (opts.json) { if (opts.json) {
stdout.write( stdout.write(
JSON.stringify( JSON.stringify(files.map(fileListRow), null, 2) + "\n",
files.map((f) => ({
id: f.id,
title: f.metadata.title,
fileType: f.metadata.fileType,
creationTime: f.metadata.creationTime,
collectionID: f.collectionID,
})),
null,
2,
) + "\n",
); );
} else { } else {
for (const f of files) { for (const file of files) {
stdout.write( stdout.write(fileListLine(file) + "\n");
`${f.id}\t${f.metadata.fileType}\t${f.metadata.title}\n`,
);
} }
} }
finish(lib, 0);
}); });
program program
@@ -196,15 +261,8 @@ program
.description("Download and decrypt a single file") .description("Download and decrypt a single file")
.argument("<fileID>", "File ID (from `quak files`)") .argument("<fileID>", "File ID (from `quak files`)")
.option("--out <path>", "Output file path") .option("--out <path>", "Output file path")
.option( .option("--collection <id>", "Accepted for compatibility; ignored")
"--collection <id>", .action(async (fileIDStr: string, opts: { out?: string }) => {
"Collection ID (required to look up the file key)",
)
.action(
async (
fileIDStr: string,
opts: { out?: string; collection?: string },
) => {
await init(); await init();
const client = requireSession(); const client = requireSession();
const fileID = Number(fileIDStr); const fileID = Number(fileIDStr);
@@ -213,46 +271,33 @@ program
process.exit(1); process.exit(1);
} }
const collections = await client.listCollections(); const lib = await openReadLibrary(client);
let targetCol; // Force a server round-trip so the file resolves against current state
if (opts.collection) { // (issue #36 amendment, issue #52).
targetCol = collections.find( const resolved = await freshFile(lib, fileID);
(c) => c.id === Number(opts.collection), if (!resolved) {
); stderr.write(`File ${fileID} not found\n`);
} finish(lib, 1);
// Search all collections (or the specified one) for the file
const searchCols = targetCol ? [targetCol] : collections;
for (const col of searchCols) {
const files = await client.listFiles(col.id, col.key);
const file = files.find((f) => f.id === fileID);
if (file) {
const result = await client.downloadFile(file, opts.out);
stderr.write(
`${result.bytesWritten} bytes -> ${result.path}\n`,
);
return; return;
} }
} const { photo, file } = resolved;
stderr.write(`File ${fileID} not found\n`);
process.exit(1); const result = await photo.original();
}, // Default name is the file's own title, as the pre-library CLI used
); // (not the editedName-preferring projection title) (issue #52).
const outPath = opts.out ?? originalName(file);
copyFileSync(result.path, outPath);
stderr.write(`${result.bytes} bytes -> ${outPath}\n`);
finish(lib, 0);
});
program program
.command("get-thumb") .command("get-thumb")
.description("Download and decrypt a thumbnail") .description("Download and decrypt a thumbnail")
.argument("<fileID>", "File ID (from `quak files`)") .argument("<fileID>", "File ID (from `quak files`)")
.option("--out <path>", "Output file path") .option("--out <path>", "Output file path")
.option( .option("--collection <id>", "Accepted for compatibility; ignored")
"--collection <id>", .action(async (fileIDStr: string, opts: { out?: string }) => {
"Collection ID (required to look up the file key)",
)
.action(
async (
fileIDStr: string,
opts: { out?: string; collection?: string },
) => {
await init(); await init();
const client = requireSession(); const client = requireSession();
const fileID = Number(fileIDStr); const fileID = Number(fileIDStr);
@@ -261,33 +306,25 @@ program
process.exit(1); process.exit(1);
} }
const collections = await client.listCollections(); const lib = await openReadLibrary(client);
let targetCol; // Force a server round-trip so the file resolves against current state
if (opts.collection) { // (issue #36 amendment, issue #52).
targetCol = collections.find( const resolved = await freshFile(lib, fileID);
(c) => c.id === Number(opts.collection), if (!resolved) {
); stderr.write(`File ${fileID} not found\n`);
} finish(lib, 1);
const searchCols = targetCol ? [targetCol] : collections;
for (const col of searchCols) {
const files = await client.listFiles(col.id, col.key);
const file = files.find((f) => f.id === fileID);
if (file) {
const result = await client.downloadThumbnail(
file,
opts.out,
);
stderr.write(
`${result.bytesWritten} bytes -> ${result.path}\n`,
);
return; return;
} }
} const { photo, file } = resolved;
stderr.write(`File ${fileID} not found\n`);
process.exit(1); const result = await photo.thumbnail();
}, // Default name is thumb_<file's own title>, as the pre-library CLI
); // used (not the projection title) (issue #52).
const outPath = opts.out ?? thumbnailName(file);
copyFileSync(result.path, outPath);
stderr.write(`${result.bytes} bytes -> ${outPath}\n`);
finish(lib, 0);
});
program program
.command("backup-metadata") .command("backup-metadata")
@@ -303,10 +340,12 @@ program
.action(async (dir: string, opts: { exif?: boolean; all?: boolean }) => { .action(async (dir: string, opts: { exif?: boolean; all?: boolean }) => {
await init(); await init();
const client = requireSession(); const client = requireSession();
await runMetadataBackup(client, dir, { const lib = await openReadLibrary(client);
await runMetadataBackup(lib, client, dir, {
exif: opts.exif || opts.all, exif: opts.exif || opts.all,
onProgress: (msg) => stderr.write(msg + "\n"), onProgress: (msg) => stderr.write(msg + "\n"),
}); });
finish(lib, 0);
}); });
program program
@@ -321,14 +360,17 @@ program
const client = requireSession(); const client = requireSession();
stderr.write("Starting backup...\n"); stderr.write("Starting backup...\n");
const lib = await Library.open({ client, downloadDirectory: dir }); const lib = await Library.open({
client,
downloadDirectory: dir,
cacheDirectory: cacheDirOption(),
});
const result = await lib.backup({ const result = await lib.backup({
downloadDirectory: dir, downloadDirectory: dir,
onProgress: (msg) => { onProgress: (msg) => {
if (!opts.json) stderr.write(msg + "\n"); if (!opts.json) stderr.write(msg + "\n");
}, },
}); });
lib.close();
if (opts.json) { if (opts.json) {
stdout.write(JSON.stringify(result, null, 2) + "\n"); stdout.write(JSON.stringify(result, null, 2) + "\n");
@@ -348,7 +390,7 @@ program
} }
} }
process.exit(result.failed > 0 ? 1 : 0); finish(lib, result.failed > 0 ? 1 : 0);
}); });
const helper = program const helper = program
@@ -362,7 +404,8 @@ helper
.action(async (opts: { json?: boolean }) => { .action(async (opts: { json?: boolean }) => {
await init(); await init();
const client = requireSession(); const client = requireSession();
const missing = await listMissingThumbnails(client, (msg) => { const lib = await openReadLibrary(client);
const missing = await listMissingThumbnails(lib, client, (msg) => {
if (!opts.json) stderr.write(msg + "\n"); if (!opts.json) stderr.write(msg + "\n");
}); });
@@ -382,6 +425,7 @@ helper
} }
} }
} }
finish(lib, 0);
}); });
helper helper
@@ -397,44 +441,61 @@ helper
.action(async (opts: { file?: string[]; json?: boolean }) => { .action(async (opts: { file?: string[]; json?: boolean }) => {
await init(); await init();
const client = requireSession(); const client = requireSession();
const lib = await openReadLibrary(client);
let fileIDs: number[]; let fileIDs: number[];
if (opts.file && opts.file.length > 0) { if (opts.file && opts.file.length > 0) {
fileIDs = opts.file.map(Number).filter(Number.isFinite); fileIDs = opts.file.map(Number).filter(Number.isFinite);
} else { } else {
stderr.write("Scanning for missing thumbnails...\n"); stderr.write("Scanning for missing thumbnails...\n");
const missing = await listMissingThumbnails(client, (msg) => { const missing = await listMissingThumbnails(lib, client, (msg) => {
if (!opts.json) stderr.write(msg + "\n"); if (!opts.json) stderr.write(msg + "\n");
}); });
fileIDs = missing.map((m) => m.fileID); fileIDs = missing.map((m) => m.fileID);
if (fileIDs.length === 0) { if (fileIDs.length === 0) {
stderr.write("No missing thumbnails found.\n"); stderr.write("No missing thumbnails found.\n");
finish(lib, 0);
return; return;
} }
stderr.write(`Found ${fileIDs.length} file(s) to fix.\n`); stderr.write(`Found ${fileIDs.length} file(s) to fix.\n`);
} }
const results = await fixMissingThumbnails(client, fileIDs, (msg) => { const results = await fixMissingThumbnails(
lib,
client,
fileIDs,
(msg) => {
if (!opts.json) stderr.write(msg + "\n"); if (!opts.json) stderr.write(msg + "\n");
}); },
);
if (opts.json) { if (opts.json) {
stdout.write(JSON.stringify(results, null, 2) + "\n"); stdout.write(JSON.stringify(results, null, 2) + "\n");
} else { } else {
const ok = results.filter((r) => r.success).length; const fixed = results.filter((r) => r.status === "fixed").length;
const fail = results.filter((r) => !r.success).length; const skipped = results.filter(
(r) => r.status === "skipped",
).length;
const failed = results.filter((r) => r.status === "failed").length;
stderr.write(`\n--- Done ---\n`); stderr.write(`\n--- Done ---\n`);
stderr.write(` Fixed: ${ok}\n`); stderr.write(` Fixed: ${fixed}\n`);
stderr.write(` Failed: ${fail}\n`); stderr.write(` Skipped: ${skipped}\n`);
if (fail > 0) { stderr.write(` Failed: ${failed}\n`);
if (skipped > 0) {
stderr.write("\nSkipped (unsupported format):\n");
for (const r of results.filter((r) => r.status === "skipped")) {
stderr.write(` ${r.fileID}\t${r.title}\t${r.reason}\n`);
}
}
if (failed > 0) {
stderr.write("\nFailed files:\n"); stderr.write("\nFailed files:\n");
for (const r of results.filter((r) => !r.success)) { for (const r of results.filter((r) => r.status === "failed")) {
stderr.write(` ${r.fileID}\t${r.title}\t${r.error}\n`); stderr.write(` ${r.fileID}\t${r.title}\t${r.reason}\n`);
} }
} }
} }
process.exit(results.some((r) => !r.success) ? 1 : 0); finish(lib, results.some((r) => r.status === "failed") ? 1 : 0);
}); });
await init(); await init();
src/cli-output.ts
+40
View File
@@ -0,0 +1,40 @@
// How the CLI presents a file's identity in `files`, `get`, and `get-thumb`.
//
// These read the file's own decrypted metadata — the raw title and the
// creationTime in microseconds — rather than the `PhotoRecord` projection the
// rest of the library exposes. The projection prefers `editedName`/`editedTime`
// and reports time in milliseconds, which is right for a photo browser but
// would change the CLI's externally-visible output. The pre-library CLI printed
// `metadata.title` and `metadata.creationTime` and named downloads after
// `metadata.title`, and issue #52 requires that output stay byte-identical, so
// the commands shape their output from the raw `EnteFile` through here.
import type { EnteFile, FileType, Microseconds } from "./model/types.js";
// One row of `quak files --json`.
export interface FileListRow {
id: number;
title: string;
fileType: FileType;
creationTime: Microseconds;
collectionID: number;
}
export const fileListRow = (file: EnteFile): FileListRow => ({
id: file.id,
title: file.metadata.title,
fileType: file.metadata.fileType,
creationTime: file.metadata.creationTime,
collectionID: file.collectionID,
});
// One line of `quak files` in its human, tab-separated form.
export const fileListLine = (file: EnteFile): string =>
`${file.id}\t${file.metadata.fileType}\t${file.metadata.title}`;
// Default output path for `quak get` when `--out` is not given.
export const originalName = (file: EnteFile): string => file.metadata.title;
// Default output path for `quak get-thumb` when `--out` is not given.
export const thumbnailName = (file: EnteFile): string =>
`thumb_${file.metadata.title}`;
src/cli-read.ts
+62
View File
@@ -0,0 +1,62 @@
// How the CLI's read commands obtain current data.
//
// `collections`, `files --collection`, `get`, and `get-thumb` must answer for
// the account's state at the moment the command runs, not for whatever the
// local cache last happened to hold (owner amendment, issue #36). Each helper
// therefore forces a server round-trip through `Library.fresh()` and only then
// reads — so a collection, file, or metadata change made elsewhere is visible.
//
// `collections` and `files` also list in the library's own enumeration order —
// `listCollections()`/`listFiles()`, the order the pre-library CLI printed —
// rather than the `albums`/`photos` projection's newest-first order, which
// re-sorts the rows. The field values still come from each record's raw
// metadata via `cli-output.ts`.
import type { Collection, EnteFile } from "./model/types.js";
import type { Photo, PhotosAPI } from "./library/index.js";
// The slice of `Library` these helpers read. `Library` satisfies it
// structurally; a test can drive them with a stand-in that records the
// `fresh()` call and serves records in a known enumeration order.
export interface FreshReadLibrary {
fresh(): Promise<unknown>;
listCollections(): Collection[];
getCollection(id: number): Collection | undefined;
listFiles(collectionID: number): EnteFile[];
getFileByID(fileID: number): EnteFile | undefined;
photos: Pick<PhotosAPI, "byID">;
}
// Every live collection, current as of a forced refresh, in enumeration order.
export const freshCollections = async (
lib: FreshReadLibrary,
): Promise<Collection[]> => {
await lib.fresh();
return lib.listCollections();
};
// The files of one collection, current as of a forced refresh, in enumeration
// order. `undefined` (not an empty list) when the collection does not exist, so
// the caller can tell "no such collection" from "an empty collection".
export const freshFiles = async (
lib: FreshReadLibrary,
collectionID: number,
): Promise<EnteFile[] | undefined> => {
await lib.fresh();
if (!lib.getCollection(collectionID)) return undefined;
return lib.listFiles(collectionID);
};
// One file, current as of a forced refresh, resolved to both its content
// handle (`Photo`, for fetching bytes) and its raw record (`EnteFile`, for the
// default output name and field values). `undefined` when the file is unknown.
export const freshFile = async (
lib: FreshReadLibrary,
fileID: number,
): Promise<{ photo: Photo; file: EnteFile } | undefined> => {
await lib.fresh();
const photo = lib.photos.byID({ fileID });
const file = lib.getFileByID(fileID);
if (!photo || !file) return undefined;
return { photo, file };
};
src/library/index.ts
+7
View File
@@ -447,6 +447,13 @@ export class Library {
return this.store.getFile(collectionID, fileID); return this.store.getFile(collectionID, fileID);
} }
// Any membership of a file, addressed by file id alone. A file's own
// metadata (title, creationTime) is identical across the collections it
// belongs to, so this serves the point commands that hold only a fileID.
getFileByID(fileID: number): EnteFile | undefined {
return this.store.getFileByID(fileID);
}
// A synchronous, RAM-only projection of the whole library into plain // A synchronous, RAM-only projection of the whole library into plain
// records (no keys), the surface the GUI reads across IPC. Photos are // records (no keys), the surface the GUI reads across IPC. Photos are
// deduplicated to one record per file and ordered newest first. // deduplicated to one record per file and ordered newest first.
src/metadata-backup.ts
+32 -25
View File
@@ -1,15 +1,9 @@
import { import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { join } from "node:path"; import { join } from "node:path";
import { tmpdir } from "node:os";
import * as jpeg from "jpeg-js"; import * as jpeg from "jpeg-js";
import exifReader from "exif-reader"; import exifReader from "exif-reader";
import type { Client } from "./client.js"; import type { Client } from "./client.js";
import type { Library, Photo } from "./library/index.js";
import { fetchMLData } from "./mldata-fetch.js"; import { fetchMLData } from "./mldata-fetch.js";
import type { EnteFile } from "./model/types.js"; import type { EnteFile } from "./model/types.js";
@@ -104,24 +98,29 @@ const extractImageMetadata = (
} }
}; };
// Read a file's original bytes through the library's content cache and extract
// its embedded image metadata. The bytes come from `photo.original()` — the
// same on-disk cache the rest of the library fills — rather than a fresh
// per-call download to a throwaway temp file.
const extractExif = async ( const extractExif = async (
client: Client, photo: Photo,
file: EnteFile,
): Promise<Record<string, unknown> | undefined> => { ): Promise<Record<string, unknown> | undefined> => {
const tmpDir = mkdtempSync(join(tmpdir(), "quak-exif-"));
try { try {
const origPath = join(tmpDir, "original"); const { path } = await photo.original();
await client.downloadFile(file, origPath); const fileBytes = new Uint8Array(readFileSync(path));
const fileBytes = new Uint8Array(readFileSync(origPath));
return extractImageMetadata(fileBytes); return extractImageMetadata(fileBytes);
} catch { } catch {
return undefined; return undefined;
} finally {
rmSync(tmpDir, { recursive: true, force: true });
} }
}; };
// Dump every decrypted metadata layer the account holds into a directory tree
// of plain JSON: account, per-collection, and per-file records including the
// private and public magic metadata and (by default) the ML data. Collections
// and files are enumerated from the library's cache rather than a fresh server
// scan; the ML fetch and EXIF extraction are unchanged.
export const runMetadataBackup = async ( export const runMetadataBackup = async (
lib: Library,
client: Client, client: Client,
outDir: string, outDir: string,
opts?: MetadataBackupOptions, opts?: MetadataBackupOptions,
@@ -139,13 +138,19 @@ export const runMetadataBackup = async (
); );
log("Fetching collections..."); log("Fetching collections...");
const collections = await client.listCollections();
const allFiles: { file: EnteFile; colDirName: string }[] = []; // Enumerate through the library's read surface. Each album carries its
// photos, but the full decrypted `Collection`/`EnteFile` records (with the
// magic-metadata layers this dump exists to preserve) come from the
// library's by-id accessors.
const allFiles: { file: EnteFile; photo: Photo; colDirName: string }[] = [];
const fileKeys = new Map<number, Uint8Array>(); const fileKeys = new Map<number, Uint8Array>();
const seenFileIDs = new Set<number>(); const seenFileIDs = new Set<number>();
for (const col of collections) { for (const album of lib.albums.list()) {
const col = lib.getCollection(album.collectionID);
if (!col) continue;
const dirName = `${col.id}-${sanitizePath(col.name || "unnamed")}`; const dirName = `${col.id}-${sanitizePath(col.name || "unnamed")}`;
const colDir = join(outDir, "collections", dirName); const colDir = join(outDir, "collections", dirName);
mkdirSync(colDir, { recursive: true }); mkdirSync(colDir, { recursive: true });
@@ -170,11 +175,13 @@ export const runMetadataBackup = async (
); );
log(`[${col.name}] Fetching files...`); log(`[${col.name}] Fetching files...`);
const files = await client.listFiles(col.id, col.key); const photos = album.photos.list();
log(`[${col.name}] ${files.length} file(s)`); log(`[${col.name}] ${photos.length} file(s)`);
for (const file of files) { for (const photo of photos) {
allFiles.push({ file, colDirName: dirName }); const file = lib.getFile(col.id, photo.fileID);
if (!file) continue;
allFiles.push({ file, photo, colDirName: dirName });
if (!seenFileIDs.has(file.id)) { if (!seenFileIDs.has(file.id)) {
fileKeys.set(file.id, file.key); fileKeys.set(file.id, file.key);
seenFileIDs.add(file.id); seenFileIDs.add(file.id);
@@ -191,7 +198,7 @@ export const runMetadataBackup = async (
log(`Got ML data for ${mlDataMap.size} file(s)`); log(`Got ML data for ${mlDataMap.size} file(s)`);
const writtenFileIDs = new Set<number>(); const writtenFileIDs = new Set<number>();
for (const { file, colDirName } of allFiles) { for (const { file, photo, colDirName } of allFiles) {
const colDir = join(outDir, "collections", colDirName); const colDir = join(outDir, "collections", colDirName);
const fileMeta: Record<string, unknown> = { const fileMeta: Record<string, unknown> = {
@@ -210,7 +217,7 @@ export const runMetadataBackup = async (
if (wantExif && !writtenFileIDs.has(file.id)) { if (wantExif && !writtenFileIDs.has(file.id)) {
log(`[${file.metadata.title}] Extracting EXIF...`); log(`[${file.metadata.title}] Extracting EXIF...`);
const exifData = await extractExif(client, file); const exifData = await extractExif(photo);
if (exifData) fileMeta.imageMetadata = exifData; if (exifData) fileMeta.imageMetadata = exifData;
} }
writtenFileIDs.add(file.id); writtenFileIDs.add(file.id);
src/thumbnails.ts
+123 -66
View File
@@ -2,13 +2,10 @@ import { createHash } from "node:crypto";
import { readFileSync } from "node:fs"; import { readFileSync } from "node:fs";
import * as jpeg from "jpeg-js"; import * as jpeg from "jpeg-js";
import type { Client } from "./client.js"; import type { Client } from "./client.js";
import type { Library } from "./library/index.js";
import { ApiError } from "./api/client.js"; import { ApiError } from "./api/client.js";
import { encryptBlob, toBase64 } from "./crypto/index.js"; import { encryptBlob, toBase64 } from "./crypto/index.js";
import { downloadFile } from "./download/index.js";
import type { EnteFile } from "./model/types.js"; import type { EnteFile } from "./model/types.js";
import { mkdtempSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
const THUMB_MAX_DIMENSION = 720; const THUMB_MAX_DIMENSION = 720;
const THUMB_JPEG_QUALITY = 50; const THUMB_JPEG_QUALITY = 50;
@@ -20,34 +17,51 @@ export interface MissingThumbnailInfo {
reason: string; reason: string;
} }
// Three outcomes, not two. "fixed": a thumbnail was generated and uploaded.
// "failed": something went wrong (download, encode, upload) and the file still
// has no thumbnail. "skipped": the file is a format this helper cannot
// regenerate — a video, or an image that is not a baseline JPEG. Skipped is a
// deliberate, expected outcome, not an error (issue #17): the repair path is
// JPEG-only because `jpeg-js` is, and a PNG or HEIC is left for a format-aware
// tool rather than reported as a failure.
export type ThumbnailFixStatus = "fixed" | "skipped" | "failed";
export interface ThumbnailFixResult { export interface ThumbnailFixResult {
fileID: number; fileID: number;
title: string; title: string;
collection: string; collection: string;
success: boolean; status: ThumbnailFixStatus;
error?: string; // Why the file was skipped or failed; unset when it was fixed.
reason?: string;
} }
export type ProgressCallback = (message: string) => void; export type ProgressCallback = (message: string) => void;
// Enumerate every file the library knows about, newest album first, each file
// once, and report those whose server-side thumbnail is missing. "Missing" is
// only two answers: an empty body, or a 404. Any other error reaching this
// point has already exhausted its retries — a failing server, a dropped
// connection, a deadline — and says nothing about whether the thumbnail
// exists, so it is logged and the file is left unreported. That distinction is
// what stops `fix-missing-thumbnails` from regenerating and uploading over
// thumbnails that were fine all along while the CDN was briefly returning 500s.
export const listMissingThumbnails = async ( export const listMissingThumbnails = async (
lib: Library,
client: Client, client: Client,
onProgress?: ProgressCallback, onProgress?: ProgressCallback,
): Promise<MissingThumbnailInfo[]> => { ): Promise<MissingThumbnailInfo[]> => {
const log = onProgress ?? (() => {}); const log = onProgress ?? (() => {});
const api = client.getApiClient();
const missing: MissingThumbnailInfo[] = []; const missing: MissingThumbnailInfo[] = [];
const seen = new Set<number>(); const seen = new Set<number>();
const collections = await client.listCollections(); for (const album of lib.albums.list()) {
for (const col of collections) { log(`[${album.name}] Checking thumbnails...`);
log(`[${col.name}] Checking thumbnails...`); for (const photo of album.photos.list()) {
const files = await client.listFiles(col.id, col.key); if (seen.has(photo.fileID)) continue;
for (const file of files) { seen.add(photo.fileID);
if (seen.has(file.id)) continue;
seen.add(file.id);
try { try {
const api = client.getApiClient(); const stream = await api.getThumbnailStream(photo.fileID);
const stream = await api.getThumbnailStream(file.id);
const reader = stream.getReader(); const reader = stream.getReader();
let totalBytes = 0; let totalBytes = 0;
for (;;) { for (;;) {
@@ -57,35 +71,23 @@ export const listMissingThumbnails = async (
} }
if (totalBytes === 0) { if (totalBytes === 0) {
missing.push({ missing.push({
fileID: file.id, fileID: photo.fileID,
title: file.metadata.title, title: photo.title,
collection: col.name, collection: album.name,
reason: "empty thumbnail (0 bytes)", reason: "empty thumbnail (0 bytes)",
}); });
} }
} catch (err) { } catch (err) {
// A 404 is the server stating the thumbnail is not there:
// that, and an empty body, are the only two answers that mean
// "missing". Anything else reaching this point is a failure
// that already exhausted its retries — a failing server, a
// dropped connection, a deadline — and says nothing about
// whether the thumbnail exists.
//
// The distinction is what stops `helper
// fix-missing-thumbnails` from downloading originals,
// regenerating thumbnails and uploading them over thumbnails
// that were fine all along, because the CDN was briefly
// returning 500s while this ran.
if (err instanceof ApiError && err.status === 404) { if (err instanceof ApiError && err.status === 404) {
missing.push({ missing.push({
fileID: file.id, fileID: photo.fileID,
title: file.metadata.title, title: photo.title,
collection: col.name, collection: album.name,
reason: "thumbnail not found (HTTP 404)", reason: "thumbnail not found (HTTP 404)",
}); });
} else { } else {
log( log(
`[${col.name}] Could not check ${file.metadata.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`, `[${album.name}] Could not check ${photo.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`,
); );
} }
} }
@@ -94,7 +96,7 @@ export const listMissingThumbnails = async (
return missing; return missing;
}; };
// Bilinear resize of RGBA pixel buffer // Bilinear resize of an RGBA pixel buffer.
const resizeRGBA = ( const resizeRGBA = (
src: Uint8Array, src: Uint8Array,
srcW: number, srcW: number,
@@ -161,7 +163,33 @@ const generateThumbnail = (fileBytes: Uint8Array): Uint8Array => {
return new Uint8Array(encoded.data); return new Uint8Array(encoded.data);
}; };
// A baseline/JFIF JPEG starts with the SOI marker 0xFFD8. `jpeg-js` decodes
// only JPEG, so this signature check is what separates a file the helper can
// regenerate from one it must skip: a PNG, HEIC, or the odd non-image byte
// stream all fail this and are reported as skipped rather than crashing the
// decoder into an opaque failure (issue #17).
const isJpeg = (bytes: Uint8Array): boolean =>
bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xd8;
// The reason a file cannot have a JPEG thumbnail regenerated for it from its
// metadata alone, before any bytes are fetched, or undefined when it might. A
// non-image (video, live photo) is unsupported outright; a still image still
// has to be checked against its actual bytes once downloaded.
const unsupportedByType = (file: EnteFile): string | undefined => {
if (file.metadata.fileType !== "image") {
return `unsupported file type: ${file.metadata.fileType} (only JPEG images can be regenerated)`;
}
return undefined;
};
// Regenerate and upload a thumbnail for each requested file. Originals are read
// through the library's content cache (`photo.original()`); the generated
// thumbnail is JPEG-encoded, encrypted under the file's own key, and registered
// with the server — the encrypt-and-upload path is unchanged. Each file is
// resolved to one outcome (fixed / skipped / failed) and a failure on one file
// never stops the others.
export const fixMissingThumbnails = async ( export const fixMissingThumbnails = async (
lib: Library,
client: Client, client: Client,
fileIDs: number[], fileIDs: number[],
onProgress?: ProgressCallback, onProgress?: ProgressCallback,
@@ -170,19 +198,24 @@ export const fixMissingThumbnails = async (
const results: ThumbnailFixResult[] = []; const results: ThumbnailFixResult[] = [];
const api = client.getApiClient(); const api = client.getApiClient();
const collections = await client.listCollections(); // Resolve each requested fileID to its file record and owning album by
// enumerating the library, each file taken from the first album that holds
// it. The raw `EnteFile` carries the per-file key the thumbnail is
// encrypted under, which the projected records deliberately do not.
const wanted = new Set(fileIDs);
const fileMap = new Map< const fileMap = new Map<
number, number,
{ file: EnteFile; collectionName: string } { file: EnteFile; collectionName: string }
>(); >();
for (const album of lib.albums.list()) {
for (const col of collections) { for (const photo of album.photos.list()) {
const files = await client.listFiles(col.id, col.key); if (!wanted.has(photo.fileID) || fileMap.has(photo.fileID))
for (const file of files) { continue;
if (fileIDs.includes(file.id) && !fileMap.has(file.id)) { const file = lib.getFile(album.collectionID, photo.fileID);
fileMap.set(file.id, { if (file) {
fileMap.set(photo.fileID, {
file, file,
collectionName: col.name, collectionName: album.name,
}); });
} }
} }
@@ -195,33 +228,61 @@ export const fixMissingThumbnails = async (
fileID, fileID,
title: "unknown", title: "unknown",
collection: "unknown", collection: "unknown",
success: false, status: "failed",
error: "file not found in any collection", reason: "file not found in any collection",
}); });
continue; continue;
} }
const { file, collectionName } = entry; const { file, collectionName } = entry;
const tmpDir = mkdtempSync(join(tmpdir(), "quak-thumb-")); const title = file.metadata.title;
const typeReason = unsupportedByType(file);
if (typeReason) {
log(`[${collectionName}] Skipping ${title}: ${typeReason}`);
results.push({
fileID,
title,
collection: collectionName,
status: "skipped",
reason: typeReason,
});
continue;
}
try { try {
log( const photo = lib.photos.byID({ fileID });
`[${collectionName}] Downloading ${file.metadata.title} for thumbnail generation...`, if (!photo) {
); throw new Error("file not present in the library cache");
const origPath = join(tmpDir, "original"); }
await downloadFile(api, file, origPath);
log( log(
`[${collectionName}] Generating thumbnail for ${file.metadata.title}...`, `[${collectionName}] Downloading ${title} for thumbnail generation...`,
); );
const fileBytes = readFileSync(origPath); const { path } = await photo.original();
const thumbJpeg = generateThumbnail(new Uint8Array(fileBytes)); const fileBytes = new Uint8Array(readFileSync(path));
if (!isJpeg(fileBytes)) {
const reason =
"unsupported image format (only baseline JPEG can be regenerated)";
log(`[${collectionName}] Skipping ${title}: ${reason}`);
results.push({
fileID,
title,
collection: collectionName,
status: "skipped",
reason,
});
continue;
}
log(`[${collectionName}] Generating thumbnail for ${title}...`);
const thumbJpeg = generateThumbnail(fileBytes);
log( log(
`[${collectionName}] Encrypting and uploading thumbnail (${thumbJpeg.length} bytes)...`, `[${collectionName}] Encrypting and uploading thumbnail (${thumbJpeg.length} bytes)...`,
); );
const { header, ciphertext } = encryptBlob(thumbJpeg, file.key); const { header, ciphertext } = encryptBlob(thumbJpeg, file.key);
const md5 = createHash("md5").update(ciphertext).digest("base64"); const md5 = createHash("md5").update(ciphertext).digest("base64");
const { objectKey, url } = await api.getUploadURL( const { objectKey, url } = await api.getUploadURL(
ciphertext.length, ciphertext.length,
@@ -230,28 +291,24 @@ export const fixMissingThumbnails = async (
await api.putFile(url, ciphertext); await api.putFile(url, ciphertext);
await api.updateThumbnail(file.id, objectKey, toBase64(header)); await api.updateThumbnail(file.id, objectKey, toBase64(header));
log( log(`[${collectionName}] Thumbnail uploaded for ${title}`);
`[${collectionName}] Thumbnail uploaded for ${file.metadata.title}`,
);
results.push({ results.push({
fileID, fileID,
title: file.metadata.title, title,
collection: collectionName, collection: collectionName,
success: true, status: "fixed",
}); });
} catch (err) { } catch (err) {
log( log(
`[${collectionName}] FAILED ${file.metadata.title}: ${err instanceof Error ? err.message : err}`, `[${collectionName}] FAILED ${title}: ${err instanceof Error ? err.message : err}`,
); );
results.push({ results.push({
fileID, fileID,
title: file.metadata.title, title,
collection: collectionName, collection: collectionName,
success: false, status: "failed",
error: err instanceof Error ? err.message : String(err), reason: err instanceof Error ? err.message : String(err),
}); });
} finally {
rmSync(tmpDir, { recursive: true, force: true });
} }
} }
test/cli/metadata-backup.test.ts
+61 -61
View File
@@ -2,13 +2,17 @@
* Tests for `quak backup-metadata <dir>`. * Tests for `quak backup-metadata <dir>`.
* *
* This command dumps all decrypted account metadata into a directory * This command dumps all decrypted account metadata into a directory
* tree of plain JSON files, without downloading any file content. It * tree of plain JSON files, without downloading any file content (unless
* is fast (no multi-megabyte downloads) and produces a complete * `--exif` is given). It is fast and produces a complete plaintext record of
* plaintext record of every collection name, file title, creation * every collection name, file title, creation date, GPS coordinate, camera
* date, GPS coordinate, camera model, caption, face label, and any * model, caption, face label, and any other metadata the Ente clients have
* other metadata the Ente clients have attached. * attached.
* *
* Layout: * As of issue #52 it runs on the library API: `runMetadataBackup(lib, client,
* dir)` enumerates collections and files from the library's cache rather than
* scanning the client directly, and `--exif` reads each original through the
* library's content cache (`photo.original()`). The ML fetch is unchanged. The
* output tree is identical:
* *
* <dir>/ * <dir>/
* account.json { email, userID } * account.json { email, userID }
@@ -44,7 +48,11 @@ import {
} from "../../src/crypto/index.js"; } from "../../src/crypto/index.js";
import * as jpegJs from "jpeg-js"; import * as jpegJs from "jpeg-js";
import { Client } from "../../src/client.js"; import { Client } from "../../src/client.js";
import { runMetadataBackup } from "../../src/metadata-backup.js"; import { Library } from "../../src/library/index.js";
import {
runMetadataBackup,
type MetadataBackupOptions,
} from "../../src/metadata-backup.js";
import type { KeyAttributes } from "../../src/auth/types.js"; import type { KeyAttributes } from "../../src/auth/types.js";
const TEST_EMAIL = "metabackup@example.com"; const TEST_EMAIL = "metabackup@example.com";
@@ -431,16 +439,50 @@ afterAll(() => {
rmSync(testDir, { recursive: true, force: true }); rmSync(testDir, { recursive: true, force: true });
}); });
describe("quak backup-metadata", () => { // Log in against the mock and open a library over its cache. The point commands
it("writes account.json with email and userID", async () => { // open the library with the background precache off and a long refresh interval;
const outDir = join(testDir, "full"); // the same here keeps the test deterministic (no thumbnail/original prefetch it
// did not ask for, no second refresh mid-test).
const openLib = async (client: Client): Promise<Library> =>
Library.open({
// The library client omits `fetchMLData`, matching how the CLI opens
// point commands: `runMetadataBackup` fetches ML data itself through
// the client, so the library's background backfill would only be a
// redundant second pass over the same endpoint.
client: {
whoami: () => client.whoami(),
collectionsSince: (args) => client.collectionsSince(args),
filesSince: (args) => client.filesSince(args),
contentSource: () => client.contentSource(),
},
cacheDirectory: mkdtempSync(join(testDir, "cache-")),
refreshIntervalSeconds: 3600,
precacheThumbnails: false,
precacheOriginals: false,
});
// Run one metadata backup end to end: fresh client, fresh library, then close.
const runBackup = async (
outDir: string,
opts?: MetadataBackupOptions,
): Promise<void> => {
const client = await Client.login({ const client = await Client.login({
email: TEST_EMAIL, email: TEST_EMAIL,
password: TEST_PASSWORD, password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) }, apiOptions: { fetch: buildMetaFetch(mock) },
}); });
const lib = await openLib(client);
try {
await runMetadataBackup(lib, client, outDir, opts);
} finally {
lib.close();
}
};
await runMetadataBackup(client, outDir); describe("quak backup-metadata", () => {
it("writes account.json with email and userID", async () => {
const outDir = join(testDir, "full");
await runBackup(outDir);
const account = JSON.parse( const account = JSON.parse(
readFileSync(join(outDir, "account.json"), "utf-8"), readFileSync(join(outDir, "account.json"), "utf-8"),
@@ -451,13 +493,7 @@ describe("quak backup-metadata", () => {
it("creates per-collection directories with _collection.json", async () => { it("creates per-collection directories with _collection.json", async () => {
const outDir = join(testDir, "collections"); const outDir = join(testDir, "collections");
const client = await Client.login({ await runBackup(outDir);
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const collDirs = readdirSync(join(outDir, "collections")); const collDirs = readdirSync(join(outDir, "collections"));
expect(collDirs.length).toBe(2); expect(collDirs.length).toBe(2);
@@ -478,13 +514,7 @@ describe("quak backup-metadata", () => {
it("decrypts collection-level pubMagicMetadata", async () => { it("decrypts collection-level pubMagicMetadata", async () => {
const outDir = join(testDir, "coll-magic"); const outDir = join(testDir, "coll-magic");
const client = await Client.login({ await runBackup(outDir);
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const collDirs = readdirSync(join(outDir, "collections")); const collDirs = readdirSync(join(outDir, "collections"));
const vacDir = collDirs.find((d) => d.includes("Vacation"))!; const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
@@ -501,13 +531,7 @@ describe("quak backup-metadata", () => {
it("writes per-file JSON with all three metadata layers", async () => { it("writes per-file JSON with all three metadata layers", async () => {
const outDir = join(testDir, "file-meta"); const outDir = join(testDir, "file-meta");
const client = await Client.login({ await runBackup(outDir);
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const collDirs = readdirSync(join(outDir, "collections")); const collDirs = readdirSync(join(outDir, "collections"));
const vacDir = collDirs.find((d) => d.includes("Vacation"))!; const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
@@ -526,13 +550,7 @@ describe("quak backup-metadata", () => {
it("handles files with no magic metadata gracefully", async () => { it("handles files with no magic metadata gracefully", async () => {
const outDir = join(testDir, "no-magic"); const outDir = join(testDir, "no-magic");
const client = await Client.login({ await runBackup(outDir);
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const collDirs = readdirSync(join(outDir, "collections")); const collDirs = readdirSync(join(outDir, "collections"));
const workDir = collDirs.find((d) => d.includes("Work"))!; const workDir = collDirs.find((d) => d.includes("Work"))!;
@@ -550,14 +568,8 @@ describe("quak backup-metadata", () => {
it("is incremental: second run does not fail", async () => { it("is incremental: second run does not fail", async () => {
const outDir = join(testDir, "incremental"); const outDir = join(testDir, "incremental");
const client = await Client.login({ await runBackup(outDir);
email: TEST_EMAIL, await runBackup(outDir);
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
await runMetadataBackup(client, outDir);
const account = JSON.parse( const account = JSON.parse(
readFileSync(join(outDir, "account.json"), "utf-8"), readFileSync(join(outDir, "account.json"), "utf-8"),
@@ -567,13 +579,7 @@ describe("quak backup-metadata", () => {
it("fetches and decrypts ML data by default", async () => { it("fetches and decrypts ML data by default", async () => {
const outDir = join(testDir, "ml-data"); const outDir = join(testDir, "ml-data");
const client = await Client.login({ await runBackup(outDir);
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir);
const collDirs = readdirSync(join(outDir, "collections")); const collDirs = readdirSync(join(outDir, "collections"));
const vacDir = collDirs.find((d) => d.includes("Vacation"))!; const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
@@ -597,13 +603,7 @@ describe("quak backup-metadata", () => {
it("extracts EXIF from downloaded files when --exif is set", async () => { it("extracts EXIF from downloaded files when --exif is set", async () => {
const outDir = join(testDir, "exif-data"); const outDir = join(testDir, "exif-data");
const client = await Client.login({ await runBackup(outDir, { exif: true });
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildMetaFetch(mock) },
});
await runMetadataBackup(client, outDir, { exif: true });
const collDirs = readdirSync(join(outDir, "collections")); const collDirs = readdirSync(join(outDir, "collections"));
const vacDir = collDirs.find((d) => d.includes("Vacation"))!; const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
test/cli/output.test.ts
+76
View File
@@ -0,0 +1,76 @@
// The CLI presents a file by its own decrypted metadata, not the PhotoRecord
// projection (issue #52). For a renamed file the two disagree: the projection
// prefers `editedName` and reports `editedTime` in milliseconds, while the CLI
// must print the raw `metadata.title` and `metadata.creationTime` (microseconds)
// and name downloads after the raw title, byte-identical to the pre-library CLI.
//
// This locks in that contrast: the shared output helpers emit the raw values,
// and the projection of the same file emits the edited ones — so a regression
// that re-sourced the CLI from the projection would fail here.
import { describe, it, expect } from "vitest";
import {
fileListRow,
fileListLine,
originalName,
thumbnailName,
} from "../../src/cli-output.js";
import { deriveRecords } from "../../src/library/records.js";
import type { EnteFile } from "../../src/model/types.js";
// Microseconds, as Ente stores times.
const RAW_CREATION = 1700000000000000;
const EDITED_TIME = 1710000000000000;
const RAW_TITLE = "IMG_0001.HEIC";
const EDITED_NAME = "Sunset.heic";
// A file the user has renamed and re-dated: basic metadata holds the original
// title and capture time; public magic metadata holds the edits.
const renamedFile: EnteFile = {
id: 100,
collectionID: 10,
ownerID: 42,
key: new Uint8Array(),
metadata: {
title: RAW_TITLE,
fileType: "image",
creationTime: RAW_CREATION,
modificationTime: RAW_CREATION,
},
pubMagicMetadata: { editedName: EDITED_NAME, editedTime: EDITED_TIME },
file: { decryptionHeader: "" },
thumbnail: { decryptionHeader: "" },
updationTime: RAW_CREATION,
};
describe("CLI file output (issue #52)", () => {
it("emits the raw title and microsecond creationTime for --json", () => {
expect(fileListRow(renamedFile)).toEqual({
id: 100,
title: RAW_TITLE,
fileType: "image",
creationTime: RAW_CREATION,
collectionID: 10,
});
});
it("emits the raw title in the human column", () => {
expect(fileListLine(renamedFile)).toBe(`100\timage\t${RAW_TITLE}`);
});
it("names downloads after the raw title", () => {
expect(originalName(renamedFile)).toBe(RAW_TITLE);
expect(thumbnailName(renamedFile)).toBe(`thumb_${RAW_TITLE}`);
});
it("does not use the editedName/editedTime projection", () => {
const record = deriveRecords([], [renamedFile]).photos.get(100);
// The projection prefers the edits and reports milliseconds; the CLI
// helpers above deliberately do not.
expect(record?.title).toBe(EDITED_NAME);
expect(record?.takenAt).toBe(Math.floor(EDITED_TIME / 1000));
expect(fileListRow(renamedFile).title).not.toBe(record?.title);
expect(fileListRow(renamedFile).creationTime).not.toBe(record?.takenAt);
});
});
test/cli/read.test.ts
+195
View File
@@ -0,0 +1,195 @@
/**
* Tests for the CLI read helpers (`src/cli-read.ts`, owner amendment to
* issue #36, issue #52).
*
* The `collections`, `files`, `get`, and `get-thumb` commands must answer for
* current server state, not the local cache, so each helper forces a
* `Library.fresh()` round-trip before it reads. The stand-in library below
* serves nothing until `fresh()` has been awaited, so a helper that read
* without refreshing would come back empty and fail here.
*
* `collections` and `files` also list in the library's enumeration order
* (`listCollections`/`listFiles`) — the order the pre-library CLI printed — not
* the albums/photos projection's newest-first order. The fixtures are seeded in
* an enumeration order that a newest-first sort would rearrange, so a
* regression to the projection order would fail here too. Field values still
* come from the raw metadata via `cli-output.ts`.
*/
import { describe, it, expect } from "vitest";
import {
freshCollections,
freshFiles,
freshFile,
type FreshReadLibrary,
} from "../../src/cli-read.js";
import { fileListRow } from "../../src/cli-output.js";
import type { Photo } from "../../src/library/index.js";
import type { Collection, EnteFile } from "../../src/model/types.js";
const collection = (id: number, updationTime: number): Collection => ({
id,
ownerID: 42,
key: new Uint8Array(),
name: `album-${id}`,
type: "album",
updationTime,
isShared: false,
});
// Microseconds, as Ente stores times.
const file = (
id: number,
collectionID: number,
creationTime: number,
): EnteFile => ({
id,
collectionID,
ownerID: 42,
key: new Uint8Array(),
metadata: {
title: `file-${id}.jpg`,
fileType: "image",
creationTime,
modificationTime: creationTime,
},
file: { decryptionHeader: "" },
thumbnail: { decryptionHeader: "" },
updationTime: creationTime,
});
// A library that reveals its records only after `fresh()` has been awaited, and
// serves them in the enumeration order it was given. `photos.byID` returns a
// stand-in `Photo` carrying just the fileID the helper passes through.
class FakeLibrary implements FreshReadLibrary {
freshCalls = 0;
private refreshed = false;
constructor(
private readonly collections: Collection[],
private readonly files: EnteFile[],
) {}
async fresh(): Promise<unknown> {
this.freshCalls++;
this.refreshed = true;
return {};
}
listCollections(): Collection[] {
return this.refreshed ? this.collections : [];
}
getCollection(id: number): Collection | undefined {
return this.listCollections().find((c) => c.id === id);
}
listFiles(collectionID: number): EnteFile[] {
return this.refreshed
? this.files.filter((f) => f.collectionID === collectionID)
: [];
}
getFileByID(fileID: number): EnteFile | undefined {
if (!this.refreshed) return undefined;
return this.files.find((f) => f.id === fileID);
}
photos = {
byID: ({ fileID }: { fileID: number }): Photo | undefined => {
if (!this.refreshed) return undefined;
if (!this.files.some((f) => f.id === fileID)) return undefined;
return { fileID } as unknown as Photo;
},
};
}
describe("CLI read helpers (issue #36 amendment, issue #52)", () => {
it("freshCollections refreshes first, then lists in enumeration order", async () => {
// Enumeration order 2, 1, 3; a newest-first sort would be 3, 2, 1.
const lib = new FakeLibrary(
[collection(2, 200), collection(1, 300), collection(3, 100)],
[],
);
const rows = await freshCollections(lib);
expect(lib.freshCalls).toBe(1);
expect(rows.map((c) => c.id)).toEqual([2, 1, 3]);
// The projection's newest-first order is a different sequence, so this
// is not accidentally that order.
const newestFirst = [...rows]
.sort((a, b) => b.updationTime - a.updationTime)
.map((c) => c.id);
expect(newestFirst).toEqual([1, 2, 3]);
expect(rows.map((c) => c.id)).not.toEqual(newestFirst);
});
it("freshFiles refreshes first, lists in enumeration order, keeps raw fields", async () => {
// Enumeration order by id 10, 11, 12; creationTimes ascending, so a
// newest-first sort would reverse them.
const files = [
file(10, 1, 1_700_000_000_000_000),
file(11, 1, 1_700_000_000_000_001),
file(12, 1, 1_700_000_000_000_002),
];
const lib = new FakeLibrary([collection(1, 100)], files);
const rows = await freshFiles(lib, 1);
expect(lib.freshCalls).toBe(1);
expect(rows?.map((f) => f.id)).toEqual([10, 11, 12]);
// Field values come from raw metadata: microsecond creationTime and the
// raw title, unchanged.
expect(rows?.map(fileListRow)).toEqual([
{
id: 10,
title: "file-10.jpg",
fileType: "image",
creationTime: 1_700_000_000_000_000,
collectionID: 1,
},
{
id: 11,
title: "file-11.jpg",
fileType: "image",
creationTime: 1_700_000_000_000_001,
collectionID: 1,
},
{
id: 12,
title: "file-12.jpg",
fileType: "image",
creationTime: 1_700_000_000_000_002,
collectionID: 1,
},
]);
});
it("freshFiles returns undefined for an unknown collection", async () => {
const lib = new FakeLibrary([collection(1, 100)], []);
const rows = await freshFiles(lib, 999);
expect(lib.freshCalls).toBe(1);
expect(rows).toBeUndefined();
});
it("freshFile refreshes first, then resolves the photo and its raw record", async () => {
const f = file(10, 1, 1_700_000_000_000_000);
const lib = new FakeLibrary([collection(1, 100)], [f]);
const resolved = await freshFile(lib, 10);
expect(lib.freshCalls).toBe(1);
expect(resolved?.photo.fileID).toBe(10);
expect(resolved?.file.metadata.title).toBe("file-10.jpg");
expect(resolved?.file.metadata.creationTime).toBe(
1_700_000_000_000_000,
);
});
it("freshFile returns undefined for an unknown file", async () => {
const lib = new FakeLibrary([collection(1, 100)], []);
const resolved = await freshFile(lib, 404);
expect(lib.freshCalls).toBe(1);
expect(resolved).toBeUndefined();
});
});
test/thumbnails/thumbnails.test.ts
+175 -83
View File
@@ -6,17 +6,30 @@
* working thumbnails, others return 404 or empty bodies. The tests * working thumbnails, others return 404 or empty bodies. The tests
* verify that the detection and repair logic handles each case correctly. * verify that the detection and repair logic handles each case correctly.
* *
* As of issue #52 both helpers take an open `Library` for enumeration and the
* `Client` for the API operations that stay unchanged (the thumbnail existence
* check, and the encrypt-and-upload path). `fixMissingThumbnails` reads each
* original through the library's content cache (`photo.original()`).
*
* `fixMissingThumbnails` is the most complex function in quak: it * `fixMissingThumbnails` is the most complex function in quak: it
* downloads the original file, generates a JPEG thumbnail with jpeg-js, * downloads the original file, generates a JPEG thumbnail with jpeg-js,
* encrypts it with secretstream push, gets a presigned upload URL, * encrypts it with secretstream push, gets a presigned upload URL,
* uploads to S3, and registers the new thumbnail with the API. The * uploads to S3, and registers the new thumbnail with the API. The
* test verifies each step actually happened and the uploaded data is * test verifies each step actually happened and the uploaded data is
* a valid encrypted blob that decrypts to a JPEG. * a valid encrypted blob that decrypts to a JPEG.
*
* It regenerates thumbnails for baseline JPEGs only, because `jpeg-js` decodes
* only JPEG. A non-JPEG image (PNG, HEIC) or a video is reported as "skipped
* (unsupported)" rather than crashing the decoder into an opaque failure
* (issue #17); the mixed test below locks that distinction down.
*/ */
import { existsSync, mkdtempSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import sodium from "libsodium-wrappers-sumo"; import sodium from "libsodium-wrappers-sumo";
import * as jpegJs from "jpeg-js"; import * as jpegJs from "jpeg-js";
import { beforeAll, describe, expect, it } from "vitest"; import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { import {
init, init,
toBase64, toBase64,
@@ -27,6 +40,7 @@ import {
} from "../../src/crypto/index.js"; } from "../../src/crypto/index.js";
import { SRP, SrpServer } from "fast-srp-hap"; import { SRP, SrpServer } from "fast-srp-hap";
import { Client } from "../../src/client.js"; import { Client } from "../../src/client.js";
import { Library } from "../../src/library/index.js";
import { import {
listMissingThumbnails, listMissingThumbnails,
fixMissingThumbnails, fixMissingThumbnails,
@@ -42,6 +56,7 @@ const TEST_EMAIL = "thumb@example.com";
const TEST_PASSWORD = "thumbpass"; const TEST_PASSWORD = "thumbpass";
const TEST_OPS = 2; const TEST_OPS = 2;
const TEST_MEM = 64 * 1024 * 1024; const TEST_MEM = 64 * 1024 * 1024;
const TEST_TIME = 1700000000000000;
interface ThumbMockState { interface ThumbMockState {
verifier: Buffer; verifier: Buffer;
@@ -63,8 +78,17 @@ interface ThumbMockState {
} }
let mock: ThumbMockState; let mock: ThumbMockState;
let tmpRoot: string;
const buildThumbMock = async (): Promise<ThumbMockState> => { // PNG signature bytes — enough for `fixMissingThumbnails` to recognise a
// non-JPEG image and skip it. It need not be a decodable PNG.
const PNG_BYTES = new Uint8Array([
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
]);
const buildThumbMock = async (opts?: {
extraFormats?: boolean;
}): Promise<ThumbMockState> => {
const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES); const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM); const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM);
const loginSubKeyBytes = deriveLoginSubkey(kek); const loginSubKeyBytes = deriveLoginSubkey(kek);
@@ -102,7 +126,6 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
opsLimit: TEST_OPS, opsLimit: TEST_OPS,
}; };
// One collection with 3 files: ok thumbnail, empty thumbnail, 404 thumbnail
const collKey = sodium.crypto_secretbox_keygen(); const collKey = sodium.crypto_secretbox_keygen();
const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
const encCK = sodium.crypto_secretbox_easy(collKey, ckN, masterKey); const encCK = sodium.crypto_secretbox_easy(collKey, ckN, masterKey);
@@ -118,10 +141,11 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
encryptedName: toBase64(encCN), encryptedName: toBase64(encCN),
nameDecryptionNonce: toBase64(cnN), nameDecryptionNonce: toBase64(cnN),
type: "album", type: "album",
updationTime: 1700000000000000, updationTime: TEST_TIME,
}; };
// Generate a real tiny JPEG via jpeg-js // Generate a real tiny JPEG via jpeg-js, used as the encrypted body of the
// JPEG files so a repair actually decodes and re-encodes real pixels.
const w = 100; const w = 100;
const h = 80; const h = 80;
const pixels = new Uint8Array(w * h * 4); const pixels = new Uint8Array(w * h * 4);
@@ -131,26 +155,32 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
pixels[i + 2] = 0; // B pixels[i + 2] = 0; // B
pixels[i + 3] = 255; // A pixels[i + 3] = 255; // A
} }
const tinyJpeg = jpegJs.encode( const tinyJpeg = new Uint8Array(
{ data: pixels, width: w, height: h }, jpegJs.encode({ data: pixels, width: w, height: h }, 80).data,
80, );
).data;
const fileKeys: Record<number, Uint8Array> = {}; const fileKeys: Record<number, Uint8Array> = {};
const fileCiphertexts: Record<number, Uint8Array> = {}; const fileCiphertexts: Record<number, Uint8Array> = {};
const rawFiles: Record<string, unknown>[] = [];
for (const fileID of [100, 101, 102]) { // Build one raw file record: encrypt its metadata and its body under a
// fresh per-file key, and record the key and ciphertext for the mock to
// serve and for the test to verify against.
const makeRawFile = (
fileID: number,
fileType: number,
title: string,
body: Uint8Array,
): Record<string, unknown> => {
const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen(); const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen();
fileKeys[fileID] = fk; fileKeys[fileID] = fk;
const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey); const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey);
const meta = JSON.stringify({ const meta = JSON.stringify({
title: `file-${fileID}.jpg`, title,
fileType: 0, fileType,
creationTime: 1700000000000000, creationTime: TEST_TIME,
modificationTime: 1700000000000000, modificationTime: TEST_TIME,
}); });
const metaPush = const metaPush =
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
@@ -161,18 +191,17 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
); );
// Encrypt the tiny JPEG as the file body
const filePush = const filePush =
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
const encFile = sodium.crypto_secretstream_xchacha20poly1305_push( const encFile = sodium.crypto_secretstream_xchacha20poly1305_push(
filePush.state, filePush.state,
new Uint8Array(tinyJpeg), body,
null, null,
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
); );
fileCiphertexts[fileID] = encFile; fileCiphertexts[fileID] = encFile;
rawFiles.push({ return {
id: fileID, id: fileID,
collectionID: 1, collectionID: 1,
ownerID: 42, ownerID: 42,
@@ -186,8 +215,30 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
thumbnail: { thumbnail: {
decryptionHeader: toBase64(sodium.randombytes_buf(24)), decryptionHeader: toBase64(sodium.randombytes_buf(24)),
}, },
updationTime: 1700000000000000, updationTime: TEST_TIME,
}); };
};
// Three JPEG files: ok thumbnail, empty thumbnail, 404 thumbnail.
const rawFiles: Record<string, unknown>[] = [];
for (const fileID of [100, 101, 102]) {
rawFiles.push(makeRawFile(fileID, 0, `file-${fileID}.jpg`, tinyJpeg));
}
const thumbnailBehavior: Record<number, "ok" | "empty" | "404" | "500"> = {
100: "ok",
101: "empty",
102: "404",
};
// For the issue #17 mixed test: a non-JPEG image and a video, both with a
// missing (404) thumbnail so they surface in the missing list too.
if (opts?.extraFormats) {
rawFiles.push(makeRawFile(103, 0, "file-103.png", PNG_BYTES));
rawFiles.push(
makeRawFile(104, 1, "file-104.mp4", new Uint8Array([0, 0, 0, 1])),
);
thumbnailBehavior[103] = "404";
thumbnailBehavior[104] = "404";
} }
return { return {
@@ -206,11 +257,7 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
filesByCollection: { 1: rawFiles }, filesByCollection: { 1: rawFiles },
fileCiphertexts, fileCiphertexts,
fileKeys, fileKeys,
thumbnailBehavior: { thumbnailBehavior,
100: "ok",
101: "empty",
102: "404",
},
uploadedThumbnails: [], uploadedThumbnails: [],
}; };
}; };
@@ -381,6 +428,31 @@ const countingFetch = (
return { fetch: fake as typeof globalThis.fetch, matched: () => matched }; return { fetch: fake as typeof globalThis.fetch, matched: () => matched };
}; };
// Open a library over a mock-backed client. As the CLI does for point commands,
// the background precache is off and the refresh interval is long, and the
// library client omits `fetchMLData` so no background ML fetch runs. The real
// `Client` is still used for the API operations the helpers perform directly.
const openLib = (client: Client): Promise<Library> =>
Library.open({
client: {
whoami: () => client.whoami(),
collectionsSince: (args) => client.collectionsSince(args),
filesSince: (args) => client.filesSince(args),
contentSource: () => client.contentSource(),
},
cacheDirectory: mkdtempSync(join(tmpRoot, "cache-")),
refreshIntervalSeconds: 3600,
precacheThumbnails: false,
precacheOriginals: false,
});
const login = (fetch: typeof globalThis.fetch, retry?: RetryOptions) =>
Client.login({
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: retry ? { fetch, retry } : { fetch },
});
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Tests // Tests
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -389,17 +461,21 @@ beforeAll(async () => {
await init(); await init();
await sodium.ready; await sodium.ready;
mock = await buildThumbMock(); mock = await buildThumbMock();
tmpRoot = mkdtempSync(join(tmpdir(), "quak-thumb-test-"));
});
afterAll(() => {
if (tmpRoot && existsSync(tmpRoot))
rmSync(tmpRoot, { recursive: true, force: true });
}); });
describe("listMissingThumbnails", () => { describe("listMissingThumbnails", () => {
it("identifies files with empty and 404 thumbnails, ignores working ones", async () => { it("identifies files with empty and 404 thumbnails, ignores working ones", async () => {
const client = await Client.login({ const client = await login(buildThumbFetch(mock));
email: TEST_EMAIL, const lib = await openLib(client);
password: TEST_PASSWORD,
apiOptions: { fetch: buildThumbFetch(mock) },
});
const missing = await listMissingThumbnails(client); const missing = await listMissingThumbnails(lib, client);
lib.close();
// File 100 has a working thumbnail → not reported // File 100 has a working thumbnail → not reported
// File 101 has an empty thumbnail → reported // File 101 has an empty thumbnail → reported
@@ -436,13 +512,11 @@ describe("listMissingThumbnails", () => {
buildThumbFetch(failingMock), buildThumbFetch(failingMock),
(url) => url.includes("thumbnails.ente.io") && url.includes("102"), (url) => url.includes("thumbnails.ente.io") && url.includes("102"),
); );
const client = await Client.login({ const client = await login(counted.fetch, { ...noWait });
email: TEST_EMAIL, const lib = await openLib(client);
password: TEST_PASSWORD,
apiOptions: { fetch: counted.fetch, retry: { ...noWait } },
});
const missing = await listMissingThumbnails(client); const missing = await listMissingThumbnails(lib, client);
lib.close();
// Only the genuinely empty thumbnail is reported. // Only the genuinely empty thumbnail is reported.
expect(missing.map((m) => m.fileID)).toEqual([101]); expect(missing.map((m) => m.fileID)).toEqual([101]);
@@ -475,13 +549,11 @@ describe("listMissingThumbnails", () => {
return inner(input, init); return inner(input, init);
}) as typeof globalThis.fetch; }) as typeof globalThis.fetch;
const client = await Client.login({ const client = await login(fetch, { ...noWait });
email: TEST_EMAIL, const lib = await openLib(client);
password: TEST_PASSWORD,
apiOptions: { fetch, retry: { ...noWait } },
});
const missing = await listMissingThumbnails(client); const missing = await listMissingThumbnails(lib, client);
lib.close();
expect(missing.map((m) => m.fileID)).toEqual([101]); expect(missing.map((m) => m.fileID)).toEqual([101]);
expect(thumbRequests).toBe(4); expect(thumbRequests).toBe(4);
@@ -500,13 +572,11 @@ describe("listMissingThumbnails", () => {
mockWithDupes.filesByCollection[2] = mockWithDupes.filesByCollection[2] =
mockWithDupes.filesByCollection[1]!; mockWithDupes.filesByCollection[1]!;
const client = await Client.login({ const client = await login(buildThumbFetch(mockWithDupes));
email: TEST_EMAIL, const lib = await openLib(client);
password: TEST_PASSWORD,
apiOptions: { fetch: buildThumbFetch(mockWithDupes) },
});
const missing = await listMissingThumbnails(client); const missing = await listMissingThumbnails(lib, client);
lib.close();
// Should still be 2, not 4 (each file checked only once) // Should still be 2, not 4 (each file checked only once)
expect(missing.length).toBe(2); expect(missing.length).toBe(2);
@@ -516,16 +586,14 @@ describe("listMissingThumbnails", () => {
describe("fixMissingThumbnails", () => { describe("fixMissingThumbnails", () => {
it("downloads original, generates thumbnail, encrypts, uploads, and registers", async () => { it("downloads original, generates thumbnail, encrypts, uploads, and registers", async () => {
const fixMock = await buildThumbMock(); const fixMock = await buildThumbMock();
const client = await Client.login({ const client = await login(buildThumbFetch(fixMock));
email: TEST_EMAIL, const lib = await openLib(client);
password: TEST_PASSWORD,
apiOptions: { fetch: buildThumbFetch(fixMock) },
});
const results = await fixMissingThumbnails(client, [101]); const results = await fixMissingThumbnails(lib, client, [101]);
lib.close();
expect(results.length).toBe(1); expect(results.length).toBe(1);
expect(results[0]!.success).toBe(true); expect(results[0]!.status).toBe("fixed");
expect(results[0]!.fileID).toBe(101); expect(results[0]!.fileID).toBe(101);
expect(results[0]!.title).toBe("file-101.jpg"); expect(results[0]!.title).toBe("file-101.jpg");
expect(results[0]!.collection).toBe("Photos"); expect(results[0]!.collection).toBe("Photos");
@@ -555,48 +623,76 @@ describe("fixMissingThumbnails", () => {
it("reports failure for nonexistent file IDs without crashing", async () => { it("reports failure for nonexistent file IDs without crashing", async () => {
const fixMock = await buildThumbMock(); const fixMock = await buildThumbMock();
const client = await Client.login({ const client = await login(buildThumbFetch(fixMock));
email: TEST_EMAIL, const lib = await openLib(client);
password: TEST_PASSWORD,
apiOptions: { fetch: buildThumbFetch(fixMock) },
});
const results = await fixMissingThumbnails(client, [999]); const results = await fixMissingThumbnails(lib, client, [999]);
lib.close();
expect(results.length).toBe(1); expect(results.length).toBe(1);
expect(results[0]!.success).toBe(false); expect(results[0]!.status).toBe("failed");
expect(results[0]!.fileID).toBe(999); expect(results[0]!.fileID).toBe(999);
expect(results[0]!.error).toContain("not found"); expect(results[0]!.reason).toContain("not found");
}); });
it("continues after one file fails and reports mixed results", async () => { it("continues after one file fails and reports mixed results", async () => {
const fixMock = await buildThumbMock(); const fixMock = await buildThumbMock();
// Make file 102 fail by removing its ciphertext so download fails // Make file 102 fail by removing its ciphertext so the download 404s.
delete fixMock.fileCiphertexts[102]; delete fixMock.fileCiphertexts[102];
const client = await Client.login({ const client = await login(buildThumbFetch(fixMock));
email: TEST_EMAIL, const lib = await openLib(client);
password: TEST_PASSWORD,
apiOptions: { fetch: buildThumbFetch(fixMock) },
});
const results = await fixMissingThumbnails(client, [101, 102]); const results = await fixMissingThumbnails(lib, client, [101, 102]);
lib.close();
expect(results.length).toBe(2); expect(results.length).toBe(2);
const success = results.find((r) => r.fileID === 101)!; const success = results.find((r) => r.fileID === 101)!;
const failure = results.find((r) => r.fileID === 102)!; const failure = results.find((r) => r.fileID === 102)!;
expect(success.success).toBe(true); expect(success.status).toBe("fixed");
expect(failure.success).toBe(false); expect(failure.status).toBe("failed");
});
it("skips a non-JPEG image and a video as unsupported, not failed (issue #17)", async () => {
// A PNG and a video both throw inside the JPEG decoder. The helper must
// recognise them up front and report "skipped", distinct from a genuine
// "failed", and must not upload anything for them. The JPEG in the same
// batch is still repaired.
const fixMock = await buildThumbMock({ extraFormats: true });
const client = await login(buildThumbFetch(fixMock));
const lib = await openLib(client);
const results = await fixMissingThumbnails(
lib,
client,
[101, 103, 104],
);
lib.close();
const jpeg = results.find((r) => r.fileID === 101)!;
const png = results.find((r) => r.fileID === 103)!;
const video = results.find((r) => r.fileID === 104)!;
expect(jpeg.status).toBe("fixed");
// The PNG is a still image but not a JPEG: skipped only after its bytes
// are inspected.
expect(png.status).toBe("skipped");
expect(png.reason).toContain("JPEG");
// The video is skipped from its type alone, before any download.
expect(video.status).toBe("skipped");
expect(video.reason).toContain("video");
// Only the JPEG was uploaded; the two skipped files touched no upload.
expect(fixMock.uploadedThumbnails.length).toBe(1);
expect(fixMock.uploadedThumbnails[0]!.fileID).toBe(101);
}); });
}); });
describe("Client.getApiClient", () => { describe("Client.getApiClient", () => {
it("returns the ApiClient when logged in", async () => { it("returns the ApiClient when logged in", async () => {
const client = await Client.login({ const client = await login(buildThumbFetch(mock));
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildThumbFetch(mock) },
});
const api = client.getApiClient(); const api = client.getApiClient();
expect(api).toBeDefined(); expect(api).toBeDefined();
@@ -604,11 +700,7 @@ describe("Client.getApiClient", () => {
}); });
it("throws after logout", async () => { it("throws after logout", async () => {
const client = await Client.login({ const client = await login(buildThumbFetch(mock));
email: TEST_EMAIL,
password: TEST_PASSWORD,
apiOptions: { fetch: buildThumbFetch(mock) },
});
client.logout(); client.logout();
expect(() => client.getApiClient()).toThrow(/logged out/); expect(() => client.getApiClient()).toThrow(/logged out/);