Test ML data with faces and no CLIP embedding, and name the norm

Each backup command's tests now have a file whose ML data holds faces and
no CLIP embedding; its JSON has the ML data and no keywords. The helper
that gives a vector's norm is called norm. The README names what a
video's JSON lacks, and the generator states its download size with the
Python packages counted.

Model: opus-5-5

README.md

@@ -1313,10 +1313,10 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange`
   an ML data fetch, and rebuilds the on-disk backup tree with a durable failure
   ledger. Each file's JSON holds its ML data and keywords. An image's JSON also
   holds the EXIF and XMP of its original, and its dimensions if it is a JPEG. A
-  live photo's JSON holds these for its image. A video's JSON holds none of
-  them. A fetched original is written straight to its save path and not into the
-  cache, which then counts it as present; one the cache already held is copied
-  from there. `BackupOptions`: `downloadDirectory` (falls back to the
+  live photo's JSON holds these for its image. A video's JSON holds no EXIF, XMP
+  or dimensions. A fetched original is written straight to its save path and not
+  into the cache, which then counts it as present; one the cache already held is
+  copied from there. `BackupOptions`: `downloadDirectory` (falls back to the
   library's), `includeOriginals` (default `true`), `includeThumbnails` (default
   `false`), `onlyAlbumNames`, `verify` (default `false`), `onProgress`, and
   `lock`. See Backup layout above for the tree it writes, and for the albums

bin/clip-keyword-vectors

@@ -2,7 +2,8 @@
 # Writes src/clip-keyword-vectors.ts and test/clip-keywords-sunset.json by
 # running bin/clip-keyword-vectors.py in a Python image pinned by digest, with
 # its packages pinned by hash. Run it by hand from a checkout, then `make fmt`;
-# no gate runs it. It downloads about 215 MB, most of it Ente's two CLIP models.
+# no gate runs it. Besides the Python image, it downloads about 265 MB, most of
+# it Ente's two CLIP models.
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"

src/clip-keywords.ts

@@ -16,14 +16,14 @@ import type { MLData } from "./mldata-fetch.js";
 // How many numbers a CLIP embedding from Ente's image model has.
 const EMBEDDING_LENGTH = 512;
 
-const length = (vector: number[]): number => Math.hypot(...vector);
+const norm = (vector: number[]): number => Math.hypot(...vector);
 
 const cosineSimilarity = (a: number[], b: number[]): number => {
     let dot = 0;
     a.forEach((value, i) => {
         dot += value * (b[i] ?? 0);
     });
-    return dot / (length(a) * length(b));
+    return dot / (norm(a) * norm(b));
 };
 
 // The titles of the Discover categories a file falls in, highest score first,
@@ -38,7 +38,7 @@ export const clipKeywords = (
     if (
         !embedding ||
         embedding.length !== EMBEDDING_LENGTH ||
-        length(embedding) === 0
+        norm(embedding) === 0
     ) {
         return undefined;
     }

test/cli/backup.test.ts

@@ -996,6 +996,33 @@ describe("ML data in each file's JSON", () => {
         await lib.close();
     });
 
+    // A file's ML data can hold faces and no CLIP embedding. Its JSON then has
+    // the ML data and no keywords.
+    it("writes no keywords for ML data without a CLIP embedding", async () => {
+        const faces = { face: { faces: [{ faceID: "101_1", score: 0.9 }] } };
+
+        // Gives file 101 ML data holding only faces.
+        class FacesClient extends MockClient {
+            async fetchMLData(args: { fileIDs: number[] }): Promise<{
+                payloads: Map<number, MLData>;
+                errors: Map<number, unknown>;
+            }> {
+                const payloads = new Map<number, MLData>();
+                if (args.fileIDs.includes(101)) payloads.set(101, faces);
+                return { payloads, errors: new Map() };
+            }
+        }
+        const lib = await openLibrary(stubSource(), new FacesClient());
+        const outDir = join(root, "backup");
+
+        const result = await lib.backup({ downloadDirectory: outDir });
+
+        expect(result.failed).toBe(0);
+        expect(fileJSON(outDir, 101).mlData).toEqual(faces);
+        expect(fileJSON(outDir, 101)).not.toHaveProperty("keywords");
+        await lib.close();
+    });
+
     // Ente's photos app keeps videos out of some Discover categories, Identity
     // among them. Here a photo and a video share one CLIP embedding, Identity's
     // own text embedding, so the photo gets the keyword Identity and the video

test/cli/metadata-backup.test.ts

@@ -79,6 +79,9 @@ const SUNSET_EMBEDDING: number[] = JSON.parse(
 // Identity, a Discover category of Ente's photos app that leaves out videos.
 const IDENTITY = DISCOVER_CATEGORIES.find((c) => c.title === "Identity")!;
 
+// ML data holding faces and no CLIP embedding, which file 302 has.
+const FACES_ONLY = { face: { faces: [{ faceID: "face-def", score: 0.9 }] } };
+
 let mock: MetaMockState;
 let testDir: string;
 
@@ -217,9 +220,14 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
         updationTime: 1710000000000000,
     };
 
-    // A file in coll 10 whose ML data holds only a CLIP embedding, Identity's
-    // own text embedding. `fileType` is Ente's number: 0 an image, 1 a video.
-    const identityFile = (id: number, title: string, fileType: number) => {
+    // A file in coll 10 with the ML data given. `fileType` is Ente's number: 0
+    // an image, 1 a video.
+    const vacationFile = (
+        id: number,
+        title: string,
+        fileType: number,
+        mlData: object,
+    ) => {
         const fileKey = sodium.crypto_secretstream_xchacha20poly1305_keygen();
         const { ciphertext: encKey, nonce } = encryptSecretbox(fileKey, ck1);
         const meta = encryptStreamBlob(
@@ -234,15 +242,7 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
             fileKey,
         );
         const ml = encryptBlob(
-            new Uint8Array(
-                gzipSync(
-                    Buffer.from(
-                        JSON.stringify({
-                            clip: { embedding: IDENTITY.textEmbedding },
-                        }),
-                    ),
-                ),
-            ),
+            new Uint8Array(gzipSync(Buffer.from(JSON.stringify(mlData)))),
             fileKey,
         );
         return {
@@ -270,9 +270,14 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
             },
         };
     };
-    // Files 300 and 301 in coll 10: a photo and a video.
-    const identityPhoto = identityFile(300, "passport.jpg", 0);
-    const identityVideo = identityFile(301, "passport.mp4", 1);
+    // Files 300 and 301 in coll 10: a photo and a video whose ML data holds
+    // only a CLIP embedding, Identity's own text embedding.
+    const identityML = { clip: { embedding: IDENTITY.textEmbedding } };
+    const identityPhoto = vacationFile(300, "passport.jpg", 0, identityML);
+    const identityVideo = vacationFile(301, "passport.mp4", 1, identityML);
+    // File 302 in coll 10: a photo whose ML data holds faces and no CLIP
+    // embedding.
+    const facesPhoto = vacationFile(302, "party.jpg", 0, FACES_ONLY);
 
     // Encrypt ML data for file 100 (gzipped JSON, encrypted with file key)
     const mlPayload = JSON.stringify({
@@ -341,7 +346,12 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
         encryptedToken: account.encryptedToken,
         collections: [rawColl1, rawColl2],
         filesByCollection: {
-            10: [rawFile1, identityPhoto.raw, identityVideo.raw],
+            10: [
+                rawFile1,
+                identityPhoto.raw,
+                identityVideo.raw,
+                facesPhoto.raw,
+            ],
             20: [rawFile2],
         },
         encryptedMLData: {
@@ -351,6 +361,7 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
             },
             300: identityPhoto.mlData,
             301: identityVideo.mlData,
+            302: facesPhoto.mlData,
         },
         fileCiphertexts: { 100: encFileBody1 },
     };
@@ -550,6 +561,22 @@ describe("quak backup-metadata", () => {
         expect(fileJSON(301).keywords).not.toContain("Identity");
     });
 
+    // A file's ML data can hold faces and no CLIP embedding. File 302's JSON
+    // then has the ML data and no keywords.
+    it("writes no keywords for ML data without a CLIP embedding", async () => {
+        const outDir = join(testDir, "no-clip");
+        await runBackup(outDir);
+
+        const fileMeta = JSON.parse(
+            readFileSync(
+                join(outDir, "collections", "10-Vacation", "302.json"),
+                "utf-8",
+            ),
+        );
+        expect(fileMeta.mlData).toEqual(FACES_ONLY);
+        expect(fileMeta).not.toHaveProperty("keywords");
+    });
+
     it("extracts EXIF from downloaded files when --exif is set", async () => {
         const outDir = join(testDir, "exif-data");
         await runBackup(outDir, { exif: true });

test/clip-keywords.test.ts

@@ -96,7 +96,7 @@ const SUNSET: { photo: string; sha256: string; embedding: number[] } =
     );
 
 describe("Ente's Discover categories", () => {
-    it("are the 16 of Ente's photos app, each with a text embedding of length 1", () => {
+    it("are the 16 of Ente's photos app, each with a text embedding of norm 1", () => {
         expect(DISCOVER_CATEGORIES.map((c) => c.title)).toEqual([
             "Identity",
             "Screenshots",