diff --git a/README.md b/README.md index 72b19c0..ad908dd 100644 --- a/README.md +++ b/README.md @@ -1313,10 +1313,10 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange` an ML data fetch, and rebuilds the on-disk backup tree with a durable failure ledger. Each file's JSON holds its ML data and keywords. An image's JSON also holds the EXIF and XMP of its original, and its dimensions if it is a JPEG. A - live photo's JSON holds these for its image. A video's JSON holds none of - them. A fetched original is written straight to its save path and not into the - cache, which then counts it as present; one the cache already held is copied - from there. `BackupOptions`: `downloadDirectory` (falls back to the + live photo's JSON holds these for its image. A video's JSON holds no EXIF, XMP + or dimensions. A fetched original is written straight to its save path and not + into the cache, which then counts it as present; one the cache already held is + copied from there. `BackupOptions`: `downloadDirectory` (falls back to the library's), `includeOriginals` (default `true`), `includeThumbnails` (default `false`), `onlyAlbumNames`, `verify` (default `false`), `onProgress`, and `lock`. See Backup layout above for the tree it writes, and for the albums diff --git a/bin/clip-keyword-vectors b/bin/clip-keyword-vectors index c94f737..df4a75f 100755 --- a/bin/clip-keyword-vectors +++ b/bin/clip-keyword-vectors @@ -2,7 +2,8 @@ # Writes src/clip-keyword-vectors.ts and test/clip-keywords-sunset.json by # running bin/clip-keyword-vectors.py in a Python image pinned by digest, with # its packages pinned by hash. Run it by hand from a checkout, then `make fmt`; -# no gate runs it. It downloads about 215 MB, most of it Ente's two CLIP models. +# no gate runs it. Besides the Python image, it downloads about 265 MB, most of +# it Ente's two CLIP models. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" diff --git a/src/clip-keywords.ts b/src/clip-keywords.ts index 0bc934a..f8c3a37 100644 --- a/src/clip-keywords.ts +++ b/src/clip-keywords.ts @@ -16,14 +16,14 @@ import type { MLData } from "./mldata-fetch.js"; // How many numbers a CLIP embedding from Ente's image model has. const EMBEDDING_LENGTH = 512; -const length = (vector: number[]): number => Math.hypot(...vector); +const norm = (vector: number[]): number => Math.hypot(...vector); const cosineSimilarity = (a: number[], b: number[]): number => { let dot = 0; a.forEach((value, i) => { dot += value * (b[i] ?? 0); }); - return dot / (length(a) * length(b)); + return dot / (norm(a) * norm(b)); }; // The titles of the Discover categories a file falls in, highest score first, @@ -38,7 +38,7 @@ export const clipKeywords = ( if ( !embedding || embedding.length !== EMBEDDING_LENGTH || - length(embedding) === 0 + norm(embedding) === 0 ) { return undefined; } diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 69c25cb..df203c7 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -996,6 +996,33 @@ describe("ML data in each file's JSON", () => { await lib.close(); }); + // A file's ML data can hold faces and no CLIP embedding. Its JSON then has + // the ML data and no keywords. + it("writes no keywords for ML data without a CLIP embedding", async () => { + const faces = { face: { faces: [{ faceID: "101_1", score: 0.9 }] } }; + + // Gives file 101 ML data holding only faces. + class FacesClient extends MockClient { + async fetchMLData(args: { fileIDs: number[] }): Promise<{ + payloads: Map; + errors: Map; + }> { + const payloads = new Map(); + if (args.fileIDs.includes(101)) payloads.set(101, faces); + return { payloads, errors: new Map() }; + } + } + const lib = await openLibrary(stubSource(), new FacesClient()); + const outDir = join(root, "backup"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + expect(fileJSON(outDir, 101).mlData).toEqual(faces); + expect(fileJSON(outDir, 101)).not.toHaveProperty("keywords"); + await lib.close(); + }); + // Ente's photos app keeps videos out of some Discover categories, Identity // among them. Here a photo and a video share one CLIP embedding, Identity's // own text embedding, so the photo gets the keyword Identity and the video diff --git a/test/cli/metadata-backup.test.ts b/test/cli/metadata-backup.test.ts index 886e743..c66c799 100644 --- a/test/cli/metadata-backup.test.ts +++ b/test/cli/metadata-backup.test.ts @@ -79,6 +79,9 @@ const SUNSET_EMBEDDING: number[] = JSON.parse( // Identity, a Discover category of Ente's photos app that leaves out videos. const IDENTITY = DISCOVER_CATEGORIES.find((c) => c.title === "Identity")!; +// ML data holding faces and no CLIP embedding, which file 302 has. +const FACES_ONLY = { face: { faces: [{ faceID: "face-def", score: 0.9 }] } }; + let mock: MetaMockState; let testDir: string; @@ -217,9 +220,14 @@ const buildMetaMock = async (): Promise => { updationTime: 1710000000000000, }; - // A file in coll 10 whose ML data holds only a CLIP embedding, Identity's - // own text embedding. `fileType` is Ente's number: 0 an image, 1 a video. - const identityFile = (id: number, title: string, fileType: number) => { + // A file in coll 10 with the ML data given. `fileType` is Ente's number: 0 + // an image, 1 a video. + const vacationFile = ( + id: number, + title: string, + fileType: number, + mlData: object, + ) => { const fileKey = sodium.crypto_secretstream_xchacha20poly1305_keygen(); const { ciphertext: encKey, nonce } = encryptSecretbox(fileKey, ck1); const meta = encryptStreamBlob( @@ -234,15 +242,7 @@ const buildMetaMock = async (): Promise => { fileKey, ); const ml = encryptBlob( - new Uint8Array( - gzipSync( - Buffer.from( - JSON.stringify({ - clip: { embedding: IDENTITY.textEmbedding }, - }), - ), - ), - ), + new Uint8Array(gzipSync(Buffer.from(JSON.stringify(mlData)))), fileKey, ); return { @@ -270,9 +270,14 @@ const buildMetaMock = async (): Promise => { }, }; }; - // Files 300 and 301 in coll 10: a photo and a video. - const identityPhoto = identityFile(300, "passport.jpg", 0); - const identityVideo = identityFile(301, "passport.mp4", 1); + // Files 300 and 301 in coll 10: a photo and a video whose ML data holds + // only a CLIP embedding, Identity's own text embedding. + const identityML = { clip: { embedding: IDENTITY.textEmbedding } }; + const identityPhoto = vacationFile(300, "passport.jpg", 0, identityML); + const identityVideo = vacationFile(301, "passport.mp4", 1, identityML); + // File 302 in coll 10: a photo whose ML data holds faces and no CLIP + // embedding. + const facesPhoto = vacationFile(302, "party.jpg", 0, FACES_ONLY); // Encrypt ML data for file 100 (gzipped JSON, encrypted with file key) const mlPayload = JSON.stringify({ @@ -341,7 +346,12 @@ const buildMetaMock = async (): Promise => { encryptedToken: account.encryptedToken, collections: [rawColl1, rawColl2], filesByCollection: { - 10: [rawFile1, identityPhoto.raw, identityVideo.raw], + 10: [ + rawFile1, + identityPhoto.raw, + identityVideo.raw, + facesPhoto.raw, + ], 20: [rawFile2], }, encryptedMLData: { @@ -351,6 +361,7 @@ const buildMetaMock = async (): Promise => { }, 300: identityPhoto.mlData, 301: identityVideo.mlData, + 302: facesPhoto.mlData, }, fileCiphertexts: { 100: encFileBody1 }, }; @@ -550,6 +561,22 @@ describe("quak backup-metadata", () => { expect(fileJSON(301).keywords).not.toContain("Identity"); }); + // A file's ML data can hold faces and no CLIP embedding. File 302's JSON + // then has the ML data and no keywords. + it("writes no keywords for ML data without a CLIP embedding", async () => { + const outDir = join(testDir, "no-clip"); + await runBackup(outDir); + + const fileMeta = JSON.parse( + readFileSync( + join(outDir, "collections", "10-Vacation", "302.json"), + "utf-8", + ), + ); + expect(fileMeta.mlData).toEqual(FACES_ONLY); + expect(fileMeta).not.toHaveProperty("keywords"); + }); + it("extracts EXIF from downloaded files when --exif is set", async () => { const outDir = join(testDir, "exif-data"); await runBackup(outDir, { exif: true }); diff --git a/test/clip-keywords.test.ts b/test/clip-keywords.test.ts index 1fb560a..13a6a5d 100644 --- a/test/clip-keywords.test.ts +++ b/test/clip-keywords.test.ts @@ -96,7 +96,7 @@ const SUNSET: { photo: string; sha256: string; embedding: number[] } = ); describe("Ente's Discover categories", () => { - it("are the 16 of Ente's photos app, each with a text embedding of length 1", () => { + it("are the 16 of Ente's photos app, each with a text embedding of norm 1", () => { expect(DISCOVER_CATEGORIES.map((c) => c.title)).toEqual([ "Identity", "Screenshots",