From 791f17b84ed02262a72c49063dd1399cafac51bc Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 22 Oct 2026 19:25:40 +0000 Subject: [PATCH] Test that a video gets no keyword from a category that leaves out videos quak backup and quak backup-metadata each get a photo and a video whose CLIP embedding is the text embedding of Identity, a Discover category that leaves out videos. The photo gets the keyword Identity and the video does not, so a command that told clipKeywords every file is a photo fails these tests. Model: opus-5-5 --- test/cli/backup.test.ts | 53 +++++++++++++++++++ test/cli/metadata-backup.test.ts | 89 ++++++++++++++++++++++++++++++-- 2 files changed, 139 insertions(+), 3 deletions(-) diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index f0c35cd..69c25cb 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -52,6 +52,7 @@ import { tmpdir } from "node:os"; import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { runBackup, type BackupLibrary } from "../../src/backup.js"; +import { DISCOVER_CATEGORIES } from "../../src/clip-keyword-vectors.js"; import { Library } from "../../src/library/index.js"; import type { ContentSource } from "../../src/library/content.js"; import type { CollectionsPage, FilesPage } from "../../src/client.js"; @@ -995,6 +996,58 @@ describe("ML data in each file's JSON", () => { await lib.close(); }); + // Ente's photos app keeps videos out of some Discover categories, Identity + // among them. Here a photo and a video share one CLIP embedding, Identity's + // own text embedding, so the photo gets the keyword Identity and the video + // does not. + it("keeps a video out of a category that leaves out videos", async () => { + const identity = DISCOVER_CATEGORIES.find( + (c) => c.title === "Identity", + )!; + expect(identity.showVideo).toBe(false); + const toast = file(102, 1, "toast.mp4"); + const video = { + ...toast, + metadata: { ...toast.metadata, fileType: "video" as const }, + }; + + // Serves beach.jpg and the video in the Vacation album, and gives + // each ML data holding only that embedding. + class VideoClient extends MockClient { + override async filesSince(args: { + collectionID: number; + }): Promise { + const files = + args.collectionID === 1 + ? [file(100, 1, "beach.jpg"), video] + : []; + return { files, deleted: [], cursor: 1 }; + } + async fetchMLData(args: { fileIDs: number[] }): Promise<{ + payloads: Map; + errors: Map; + }> { + const payloads = new Map(); + for (const id of args.fileIDs) { + payloads.set(id, { + clip: { embedding: identity.textEmbedding }, + }); + } + return { payloads, errors: new Map() }; + } + } + const lib = await openLibrary(stubSource(), new VideoClient()); + const outDir = join(root, "backup"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + expect(fileJSON(outDir, 100).keywords).toContain("Identity"); + expect(fileJSON(outDir, 102)).toHaveProperty("keywords"); + expect(fileJSON(outDir, 102).keywords).not.toContain("Identity"); + await lib.close(); + }); + it("gives each file with no cached ML data the reason when the fetch fails, and fails the run", async () => { const client = new MLClient(); const outDir = join(root, "backup"); diff --git a/test/cli/metadata-backup.test.ts b/test/cli/metadata-backup.test.ts index d8d28e1..886e743 100644 --- a/test/cli/metadata-backup.test.ts +++ b/test/cli/metadata-backup.test.ts @@ -41,6 +41,7 @@ import { beforeAll, afterAll, describe, expect, it, vi } from "vitest"; import { init, toBase64, encryptBlob } from "../../src/crypto/index.js"; import * as jpegJs from "jpeg-js"; import { Client } from "../../src/client.js"; +import { DISCOVER_CATEGORIES } from "../../src/clip-keyword-vectors.js"; import { Library } from "../../src/library/index.js"; import { runMetadataBackup, @@ -56,8 +57,8 @@ import { type MetaMockState, } from "../ente-server.js"; -// One file per ML data request, so the two files of the mock account are -// fetched in two requests and one of them can fail on its own. +// One file per ML data request, so each file of the mock account is fetched in +// a request of its own and one of them can fail on its own. vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({ ...(await importOriginal()), MLDATA_BATCH_SIZE: 1, @@ -75,6 +76,9 @@ const SUNSET_EMBEDDING: number[] = JSON.parse( ), ).embedding; +// Identity, a Discover category of Ente's photos app that leaves out videos. +const IDENTITY = DISCOVER_CATEGORIES.find((c) => c.title === "Identity")!; + let mock: MetaMockState; let testDir: string; @@ -213,6 +217,63 @@ const buildMetaMock = async (): Promise => { updationTime: 1710000000000000, }; + // A file in coll 10 whose ML data holds only a CLIP embedding, Identity's + // own text embedding. `fileType` is Ente's number: 0 an image, 1 a video. + const identityFile = (id: number, title: string, fileType: number) => { + const fileKey = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { ciphertext: encKey, nonce } = encryptSecretbox(fileKey, ck1); + const meta = encryptStreamBlob( + new TextEncoder().encode( + JSON.stringify({ + title, + fileType, + creationTime: 1700000000000000, + modificationTime: 1700000000000000, + }), + ), + fileKey, + ); + const ml = encryptBlob( + new Uint8Array( + gzipSync( + Buffer.from( + JSON.stringify({ + clip: { embedding: IDENTITY.textEmbedding }, + }), + ), + ), + ), + fileKey, + ); + return { + raw: { + id, + collectionID: 10, + ownerID: 42, + encryptedKey: toBase64(encKey), + keyDecryptionNonce: toBase64(nonce), + metadata: { + encryptedData: toBase64(meta.ciphertext), + decryptionHeader: toBase64(meta.header), + }, + file: { + decryptionHeader: toBase64(sodium.randombytes_buf(24)), + }, + thumbnail: { + decryptionHeader: toBase64(sodium.randombytes_buf(24)), + }, + updationTime: 1700000000000000, + }, + mlData: { + encryptedData: toBase64(ml.ciphertext), + decryptionHeader: toBase64(ml.header), + }, + }; + }; + // Files 300 and 301 in coll 10: a photo and a video. + const identityPhoto = identityFile(300, "passport.jpg", 0); + const identityVideo = identityFile(301, "passport.mp4", 1); + // Encrypt ML data for file 100 (gzipped JSON, encrypted with file key) const mlPayload = JSON.stringify({ face: { @@ -279,12 +340,17 @@ const buildMetaMock = async (): Promise => { keyAttributes: account.keyAttributes, encryptedToken: account.encryptedToken, collections: [rawColl1, rawColl2], - filesByCollection: { 10: [rawFile1], 20: [rawFile2] }, + filesByCollection: { + 10: [rawFile1, identityPhoto.raw, identityVideo.raw], + 20: [rawFile2], + }, encryptedMLData: { 100: { encryptedData: toBase64(mlCiphertext), decryptionHeader: toBase64(mlHeader), }, + 300: identityPhoto.mlData, + 301: identityVideo.mlData, }, fileCiphertexts: { 100: encFileBody1 }, }; @@ -467,6 +533,23 @@ describe("quak backup-metadata", () => { expect(fileMeta.keywords).toEqual(["Sunset"]); }); + // Ente's photos app keeps videos out of some Discover categories, Identity + // among them. Files 300 and 301, a photo and a video, share one CLIP + // embedding, Identity's own text embedding, so the photo gets the keyword + // Identity and the video does not. + it("keeps a video out of a category that leaves out videos", async () => { + expect(IDENTITY.showVideo).toBe(false); + const outDir = join(testDir, "video-keywords"); + await runBackup(outDir); + + const vacation = join(outDir, "collections", "10-Vacation"); + const fileJSON = (fileID: number) => + JSON.parse(readFileSync(join(vacation, `${fileID}.json`), "utf-8")); + expect(fileJSON(300).keywords).toContain("Identity"); + expect(fileJSON(301)).toHaveProperty("keywords"); + expect(fileJSON(301).keywords).not.toContain("Identity"); + }); + it("extracts EXIF from downloaded files when --exif is set", async () => { const outDir = join(testDir, "exif-data"); await runBackup(outDir, { exif: true });