Test that a video gets no keyword from a category that leaves out videos
quak backup and quak backup-metadata each get a photo and a video whose CLIP embedding is the text embedding of Identity, a Discover category that leaves out videos. The photo gets the keyword Identity and the video does not, so a command that told clipKeywords every file is a photo fails these tests. Model: opus-5-5
This commit is contained in:
@@ -41,6 +41,7 @@ import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
|
||||
import { init, toBase64, encryptBlob } from "../../src/crypto/index.js";
|
||||
import * as jpegJs from "jpeg-js";
|
||||
import { Client } from "../../src/client.js";
|
||||
import { DISCOVER_CATEGORIES } from "../../src/clip-keyword-vectors.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
import {
|
||||
runMetadataBackup,
|
||||
@@ -56,8 +57,8 @@ import {
|
||||
type MetaMockState,
|
||||
} from "../ente-server.js";
|
||||
|
||||
// One file per ML data request, so the two files of the mock account are
|
||||
// fetched in two requests and one of them can fail on its own.
|
||||
// One file per ML data request, so each file of the mock account is fetched in
|
||||
// a request of its own and one of them can fail on its own.
|
||||
vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../../src/mldata-fetch.js")>()),
|
||||
MLDATA_BATCH_SIZE: 1,
|
||||
@@ -75,6 +76,9 @@ const SUNSET_EMBEDDING: number[] = JSON.parse(
|
||||
),
|
||||
).embedding;
|
||||
|
||||
// Identity, a Discover category of Ente's photos app that leaves out videos.
|
||||
const IDENTITY = DISCOVER_CATEGORIES.find((c) => c.title === "Identity")!;
|
||||
|
||||
let mock: MetaMockState;
|
||||
let testDir: string;
|
||||
|
||||
@@ -213,6 +217,63 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
|
||||
updationTime: 1710000000000000,
|
||||
};
|
||||
|
||||
// A file in coll 10 whose ML data holds only a CLIP embedding, Identity's
|
||||
// own text embedding. `fileType` is Ente's number: 0 an image, 1 a video.
|
||||
const identityFile = (id: number, title: string, fileType: number) => {
|
||||
const fileKey = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
const { ciphertext: encKey, nonce } = encryptSecretbox(fileKey, ck1);
|
||||
const meta = encryptStreamBlob(
|
||||
new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
title,
|
||||
fileType,
|
||||
creationTime: 1700000000000000,
|
||||
modificationTime: 1700000000000000,
|
||||
}),
|
||||
),
|
||||
fileKey,
|
||||
);
|
||||
const ml = encryptBlob(
|
||||
new Uint8Array(
|
||||
gzipSync(
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
clip: { embedding: IDENTITY.textEmbedding },
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
fileKey,
|
||||
);
|
||||
return {
|
||||
raw: {
|
||||
id,
|
||||
collectionID: 10,
|
||||
ownerID: 42,
|
||||
encryptedKey: toBase64(encKey),
|
||||
keyDecryptionNonce: toBase64(nonce),
|
||||
metadata: {
|
||||
encryptedData: toBase64(meta.ciphertext),
|
||||
decryptionHeader: toBase64(meta.header),
|
||||
},
|
||||
file: {
|
||||
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
||||
},
|
||||
thumbnail: {
|
||||
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
||||
},
|
||||
updationTime: 1700000000000000,
|
||||
},
|
||||
mlData: {
|
||||
encryptedData: toBase64(ml.ciphertext),
|
||||
decryptionHeader: toBase64(ml.header),
|
||||
},
|
||||
};
|
||||
};
|
||||
// Files 300 and 301 in coll 10: a photo and a video.
|
||||
const identityPhoto = identityFile(300, "passport.jpg", 0);
|
||||
const identityVideo = identityFile(301, "passport.mp4", 1);
|
||||
|
||||
// Encrypt ML data for file 100 (gzipped JSON, encrypted with file key)
|
||||
const mlPayload = JSON.stringify({
|
||||
face: {
|
||||
@@ -279,12 +340,17 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
|
||||
keyAttributes: account.keyAttributes,
|
||||
encryptedToken: account.encryptedToken,
|
||||
collections: [rawColl1, rawColl2],
|
||||
filesByCollection: { 10: [rawFile1], 20: [rawFile2] },
|
||||
filesByCollection: {
|
||||
10: [rawFile1, identityPhoto.raw, identityVideo.raw],
|
||||
20: [rawFile2],
|
||||
},
|
||||
encryptedMLData: {
|
||||
100: {
|
||||
encryptedData: toBase64(mlCiphertext),
|
||||
decryptionHeader: toBase64(mlHeader),
|
||||
},
|
||||
300: identityPhoto.mlData,
|
||||
301: identityVideo.mlData,
|
||||
},
|
||||
fileCiphertexts: { 100: encFileBody1 },
|
||||
};
|
||||
@@ -467,6 +533,23 @@ describe("quak backup-metadata", () => {
|
||||
expect(fileMeta.keywords).toEqual(["Sunset"]);
|
||||
});
|
||||
|
||||
// Ente's photos app keeps videos out of some Discover categories, Identity
|
||||
// among them. Files 300 and 301, a photo and a video, share one CLIP
|
||||
// embedding, Identity's own text embedding, so the photo gets the keyword
|
||||
// Identity and the video does not.
|
||||
it("keeps a video out of a category that leaves out videos", async () => {
|
||||
expect(IDENTITY.showVideo).toBe(false);
|
||||
const outDir = join(testDir, "video-keywords");
|
||||
await runBackup(outDir);
|
||||
|
||||
const vacation = join(outDir, "collections", "10-Vacation");
|
||||
const fileJSON = (fileID: number) =>
|
||||
JSON.parse(readFileSync(join(vacation, `${fileID}.json`), "utf-8"));
|
||||
expect(fileJSON(300).keywords).toContain("Identity");
|
||||
expect(fileJSON(301)).toHaveProperty("keywords");
|
||||
expect(fileJSON(301).keywords).not.toContain("Identity");
|
||||
});
|
||||
|
||||
it("extracts EXIF from downloaded files when --exif is set", async () => {
|
||||
const outDir = join(testDir, "exif-data");
|
||||
await runBackup(outDir, { exif: true });
|
||||
|
||||
Reference in New Issue
Block a user