Test that a video gets no keyword from a category that leaves out videos

quak backup and quak backup-metadata each get a photo and a video whose CLIP
embedding is the text embedding of Identity, a Discover category that leaves
out videos. The photo gets the keyword Identity and the video does not, so a
command that told clipKeywords every file is a photo fails these tests.

Model: opus-5-5
This commit is contained in:
2026-10-22 19:25:40 +00:00
parent 5ec176550f
commit 791f17b84e
2 changed files with 139 additions and 3 deletions
+86 -3
View File
@@ -41,6 +41,7 @@ import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
import { init, toBase64, encryptBlob } from "../../src/crypto/index.js";
import * as jpegJs from "jpeg-js";
import { Client } from "../../src/client.js";
import { DISCOVER_CATEGORIES } from "../../src/clip-keyword-vectors.js";
import { Library } from "../../src/library/index.js";
import {
runMetadataBackup,
@@ -56,8 +57,8 @@ import {
type MetaMockState,
} from "../ente-server.js";
// One file per ML data request, so the two files of the mock account are
// fetched in two requests and one of them can fail on its own.
// One file per ML data request, so each file of the mock account is fetched in
// a request of its own and one of them can fail on its own.
vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../../src/mldata-fetch.js")>()),
MLDATA_BATCH_SIZE: 1,
@@ -75,6 +76,9 @@ const SUNSET_EMBEDDING: number[] = JSON.parse(
),
).embedding;
// Identity, a Discover category of Ente's photos app that leaves out videos.
const IDENTITY = DISCOVER_CATEGORIES.find((c) => c.title === "Identity")!;
let mock: MetaMockState;
let testDir: string;
@@ -213,6 +217,63 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
updationTime: 1710000000000000,
};
// A file in coll 10 whose ML data holds only a CLIP embedding, Identity's
// own text embedding. `fileType` is Ente's number: 0 an image, 1 a video.
const identityFile = (id: number, title: string, fileType: number) => {
const fileKey = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { ciphertext: encKey, nonce } = encryptSecretbox(fileKey, ck1);
const meta = encryptStreamBlob(
new TextEncoder().encode(
JSON.stringify({
title,
fileType,
creationTime: 1700000000000000,
modificationTime: 1700000000000000,
}),
),
fileKey,
);
const ml = encryptBlob(
new Uint8Array(
gzipSync(
Buffer.from(
JSON.stringify({
clip: { embedding: IDENTITY.textEmbedding },
}),
),
),
),
fileKey,
);
return {
raw: {
id,
collectionID: 10,
ownerID: 42,
encryptedKey: toBase64(encKey),
keyDecryptionNonce: toBase64(nonce),
metadata: {
encryptedData: toBase64(meta.ciphertext),
decryptionHeader: toBase64(meta.header),
},
file: {
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
},
thumbnail: {
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
},
updationTime: 1700000000000000,
},
mlData: {
encryptedData: toBase64(ml.ciphertext),
decryptionHeader: toBase64(ml.header),
},
};
};
// Files 300 and 301 in coll 10: a photo and a video.
const identityPhoto = identityFile(300, "passport.jpg", 0);
const identityVideo = identityFile(301, "passport.mp4", 1);
// Encrypt ML data for file 100 (gzipped JSON, encrypted with file key)
const mlPayload = JSON.stringify({
face: {
@@ -279,12 +340,17 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
keyAttributes: account.keyAttributes,
encryptedToken: account.encryptedToken,
collections: [rawColl1, rawColl2],
filesByCollection: { 10: [rawFile1], 20: [rawFile2] },
filesByCollection: {
10: [rawFile1, identityPhoto.raw, identityVideo.raw],
20: [rawFile2],
},
encryptedMLData: {
100: {
encryptedData: toBase64(mlCiphertext),
decryptionHeader: toBase64(mlHeader),
},
300: identityPhoto.mlData,
301: identityVideo.mlData,
},
fileCiphertexts: { 100: encFileBody1 },
};
@@ -467,6 +533,23 @@ describe("quak backup-metadata", () => {
expect(fileMeta.keywords).toEqual(["Sunset"]);
});
// Ente's photos app keeps videos out of some Discover categories, Identity
// among them. Files 300 and 301, a photo and a video, share one CLIP
// embedding, Identity's own text embedding, so the photo gets the keyword
// Identity and the video does not.
it("keeps a video out of a category that leaves out videos", async () => {
expect(IDENTITY.showVideo).toBe(false);
const outDir = join(testDir, "video-keywords");
await runBackup(outDir);
const vacation = join(outDir, "collections", "10-Vacation");
const fileJSON = (fileID: number) =>
JSON.parse(readFileSync(join(vacation, `${fileID}.json`), "utf-8"));
expect(fileJSON(300).keywords).toContain("Identity");
expect(fileJSON(301)).toHaveProperty("keywords");
expect(fileJSON(301).keywords).not.toContain("Identity");
});
it("extracts EXIF from downloaded files when --exif is set", async () => {
const outDir = join(testDir, "exif-data");
await runBackup(outDir, { exif: true });