Route every command through Library.open instead of scanning the client. collections/files read lib.albums; get/get-thumb resolve lib.photos.byID and copy the cached original/thumbnail to --out, so --collection is now accepted but ignored. backup-metadata and the thumbnail helpers enumerate through the library and read originals via photo.original(); ML fetch, EXIF, and thumbnail upload are unchanged. A new global --cache-dir sets the cache location; point commands open with the background precache off so a one-shot command never starts downloading the whole account. files, get, and get-thumb present each file from its own decrypted metadata (raw title, creationTime in microseconds) rather than the PhotoRecord projection, which prefers editedName/editedTime and milliseconds. This keeps the CLI output byte-identical to the pre-library version. Also addresses #17: fix-missing-thumbnails now reports a non-JPEG image or a video as skipped (unsupported), distinct from failed, and only a genuine failure exits non-zero. Model: opus-4-8
This commit is contained in:
@@ -2,13 +2,17 @@
|
||||
* Tests for `quak backup-metadata <dir>`.
|
||||
*
|
||||
* This command dumps all decrypted account metadata into a directory
|
||||
* tree of plain JSON files, without downloading any file content. It
|
||||
* is fast (no multi-megabyte downloads) and produces a complete
|
||||
* plaintext record of every collection name, file title, creation
|
||||
* date, GPS coordinate, camera model, caption, face label, and any
|
||||
* other metadata the Ente clients have attached.
|
||||
* tree of plain JSON files, without downloading any file content (unless
|
||||
* `--exif` is given). It is fast and produces a complete plaintext record of
|
||||
* every collection name, file title, creation date, GPS coordinate, camera
|
||||
* model, caption, face label, and any other metadata the Ente clients have
|
||||
* attached.
|
||||
*
|
||||
* Layout:
|
||||
* As of issue #52 it runs on the library API: `runMetadataBackup(lib, client,
|
||||
* dir)` enumerates collections and files from the library's cache rather than
|
||||
* scanning the client directly, and `--exif` reads each original through the
|
||||
* library's content cache (`photo.original()`). The ML fetch is unchanged. The
|
||||
* output tree is identical:
|
||||
*
|
||||
* <dir>/
|
||||
* account.json { email, userID }
|
||||
@@ -44,7 +48,11 @@ import {
|
||||
} from "../../src/crypto/index.js";
|
||||
import * as jpegJs from "jpeg-js";
|
||||
import { Client } from "../../src/client.js";
|
||||
import { runMetadataBackup } from "../../src/metadata-backup.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
import {
|
||||
runMetadataBackup,
|
||||
type MetadataBackupOptions,
|
||||
} from "../../src/metadata-backup.js";
|
||||
import type { KeyAttributes } from "../../src/auth/types.js";
|
||||
|
||||
const TEST_EMAIL = "metabackup@example.com";
|
||||
@@ -431,16 +439,50 @@ afterAll(() => {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Log in against the mock and open a library over its cache. The point commands
|
||||
// open the library with the background precache off and a long refresh interval;
|
||||
// the same here keeps the test deterministic (no thumbnail/original prefetch it
|
||||
// did not ask for, no second refresh mid-test).
|
||||
const openLib = async (client: Client): Promise<Library> =>
|
||||
Library.open({
|
||||
// The library client omits `fetchMLData`, matching how the CLI opens
|
||||
// point commands: `runMetadataBackup` fetches ML data itself through
|
||||
// the client, so the library's background backfill would only be a
|
||||
// redundant second pass over the same endpoint.
|
||||
client: {
|
||||
whoami: () => client.whoami(),
|
||||
collectionsSince: (args) => client.collectionsSince(args),
|
||||
filesSince: (args) => client.filesSince(args),
|
||||
contentSource: () => client.contentSource(),
|
||||
},
|
||||
cacheDirectory: mkdtempSync(join(testDir, "cache-")),
|
||||
refreshIntervalSeconds: 3600,
|
||||
precacheThumbnails: false,
|
||||
precacheOriginals: false,
|
||||
});
|
||||
|
||||
// Run one metadata backup end to end: fresh client, fresh library, then close.
|
||||
const runBackup = async (
|
||||
outDir: string,
|
||||
opts?: MetadataBackupOptions,
|
||||
): Promise<void> => {
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
const lib = await openLib(client);
|
||||
try {
|
||||
await runMetadataBackup(lib, client, outDir, opts);
|
||||
} finally {
|
||||
lib.close();
|
||||
}
|
||||
};
|
||||
|
||||
describe("quak backup-metadata", () => {
|
||||
it("writes account.json with email and userID", async () => {
|
||||
const outDir = join(testDir, "full");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const account = JSON.parse(
|
||||
readFileSync(join(outDir, "account.json"), "utf-8"),
|
||||
@@ -451,13 +493,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("creates per-collection directories with _collection.json", async () => {
|
||||
const outDir = join(testDir, "collections");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
expect(collDirs.length).toBe(2);
|
||||
@@ -478,13 +514,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("decrypts collection-level pubMagicMetadata", async () => {
|
||||
const outDir = join(testDir, "coll-magic");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
@@ -501,13 +531,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("writes per-file JSON with all three metadata layers", async () => {
|
||||
const outDir = join(testDir, "file-meta");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
@@ -526,13 +550,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("handles files with no magic metadata gracefully", async () => {
|
||||
const outDir = join(testDir, "no-magic");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const workDir = collDirs.find((d) => d.includes("Work"))!;
|
||||
@@ -550,14 +568,8 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("is incremental: second run does not fail", async () => {
|
||||
const outDir = join(testDir, "incremental");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const account = JSON.parse(
|
||||
readFileSync(join(outDir, "account.json"), "utf-8"),
|
||||
@@ -567,13 +579,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("fetches and decrypts ML data by default", async () => {
|
||||
const outDir = join(testDir, "ml-data");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir);
|
||||
await runBackup(outDir);
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
@@ -597,13 +603,7 @@ describe("quak backup-metadata", () => {
|
||||
|
||||
it("extracts EXIF from downloaded files when --exif is set", async () => {
|
||||
const outDir = join(testDir, "exif-data");
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||
});
|
||||
|
||||
await runMetadataBackup(client, outDir, { exif: true });
|
||||
await runBackup(outDir, { exif: true });
|
||||
|
||||
const collDirs = readdirSync(join(outDir, "collections"));
|
||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
// The CLI presents a file by its own decrypted metadata, not the PhotoRecord
|
||||
// projection (issue #52). For a renamed file the two disagree: the projection
|
||||
// prefers `editedName` and reports `editedTime` in milliseconds, while the CLI
|
||||
// must print the raw `metadata.title` and `metadata.creationTime` (microseconds)
|
||||
// and name downloads after the raw title, byte-identical to the pre-library CLI.
|
||||
//
|
||||
// This locks in that contrast: the shared output helpers emit the raw values,
|
||||
// and the projection of the same file emits the edited ones — so a regression
|
||||
// that re-sourced the CLI from the projection would fail here.
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import {
|
||||
fileListRow,
|
||||
fileListLine,
|
||||
originalName,
|
||||
thumbnailName,
|
||||
} from "../../src/cli-output.js";
|
||||
import { deriveRecords } from "../../src/library/records.js";
|
||||
import type { EnteFile } from "../../src/model/types.js";
|
||||
|
||||
// Microseconds, as Ente stores times.
|
||||
const RAW_CREATION = 1700000000000000;
|
||||
const EDITED_TIME = 1710000000000000;
|
||||
const RAW_TITLE = "IMG_0001.HEIC";
|
||||
const EDITED_NAME = "Sunset.heic";
|
||||
|
||||
// A file the user has renamed and re-dated: basic metadata holds the original
|
||||
// title and capture time; public magic metadata holds the edits.
|
||||
const renamedFile: EnteFile = {
|
||||
id: 100,
|
||||
collectionID: 10,
|
||||
ownerID: 42,
|
||||
key: new Uint8Array(),
|
||||
metadata: {
|
||||
title: RAW_TITLE,
|
||||
fileType: "image",
|
||||
creationTime: RAW_CREATION,
|
||||
modificationTime: RAW_CREATION,
|
||||
},
|
||||
pubMagicMetadata: { editedName: EDITED_NAME, editedTime: EDITED_TIME },
|
||||
file: { decryptionHeader: "" },
|
||||
thumbnail: { decryptionHeader: "" },
|
||||
updationTime: RAW_CREATION,
|
||||
};
|
||||
|
||||
describe("CLI file output (issue #52)", () => {
|
||||
it("emits the raw title and microsecond creationTime for --json", () => {
|
||||
expect(fileListRow(renamedFile)).toEqual({
|
||||
id: 100,
|
||||
title: RAW_TITLE,
|
||||
fileType: "image",
|
||||
creationTime: RAW_CREATION,
|
||||
collectionID: 10,
|
||||
});
|
||||
});
|
||||
|
||||
it("emits the raw title in the human column", () => {
|
||||
expect(fileListLine(renamedFile)).toBe(`100\timage\t${RAW_TITLE}`);
|
||||
});
|
||||
|
||||
it("names downloads after the raw title", () => {
|
||||
expect(originalName(renamedFile)).toBe(RAW_TITLE);
|
||||
expect(thumbnailName(renamedFile)).toBe(`thumb_${RAW_TITLE}`);
|
||||
});
|
||||
|
||||
it("does not use the editedName/editedTime projection", () => {
|
||||
const record = deriveRecords([], [renamedFile]).photos.get(100);
|
||||
// The projection prefers the edits and reports milliseconds; the CLI
|
||||
// helpers above deliberately do not.
|
||||
expect(record?.title).toBe(EDITED_NAME);
|
||||
expect(record?.takenAt).toBe(Math.floor(EDITED_TIME / 1000));
|
||||
expect(fileListRow(renamedFile).title).not.toBe(record?.title);
|
||||
expect(fileListRow(renamedFile).creationTime).not.toBe(record?.takenAt);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user