From 4863833f0ac1e79fefad2075027e7626831398b9 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 22 Oct 2026 18:36:30 +0000 Subject: [PATCH] Test keywords in place of the CLIP embedding in a file's JSON clipKeywords takes a file's ML data and whether it is a video, and gives the titles of the Discover categories of Ente's photos app that its CLIP embedding falls in, highest score first. The tests cover the rule with made-up vectors and Ente's real categories with the CLIP embedding of a public-domain photo of a sunset. The backup and backup-metadata tests now expect keywords in a file's JSON and no CLIP embedding in its mlData. These tests fail until the implementation lands. Model: opus-5-5 --- test/cli/backup-end-to-end.test.ts | 17 +++- test/cli/backup.test.ts | 28 +++++-- test/cli/metadata-backup.test.ts | 23 +++-- test/clip-keywords-sunset.json | 109 ++++++++++++++++++++++++ test/clip-keywords.test.ts | 129 +++++++++++++++++++++++++++++ 5 files changed, 290 insertions(+), 16 deletions(-) create mode 100644 test/clip-keywords-sunset.json create mode 100644 test/clip-keywords.test.ts diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index 46895e3..8f075f7 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -86,10 +86,17 @@ const DAY = "2024/2024-03/2024-03-01"; const TOAST = new TextEncoder().encode("a few seconds of the wedding toast"); // Ente's ML data for beach.jpg: a face it found, and the photo's CLIP -// embedding. +// embedding, here that of a photo of a sunset (see clip-keywords.test.ts). const ML_DATA = { face: { faces: [{ faceID: "100_1", score: 0.98 }] }, - clip: { embedding: [0.5, 0.6, 0.7] }, + clip: { + embedding: JSON.parse( + readFileSync( + new URL("../clip-keywords-sunset.json", import.meta.url), + "utf-8", + ), + ).embedding, + }, }; // Each file's ID, its owner, its original as uploaded, and the metadata its @@ -486,7 +493,10 @@ describe("the first backup of the account", () => { }, updationTime: TAKEN, }); - expect(beach.mlData).toEqual(ML_DATA); + // The JSON holds the ML data without its CLIP embedding, and the + // keywords the embedding gives in its place. + expect(beach.mlData).toEqual({ face: ML_DATA.face }); + expect(beach.keywords).toEqual(["Sunset"]); expect(beach.imageMetadata.exif.Make.description).toBe("Canon"); expect(beach.imageMetadata.exif.Model.description).toBe("EOS R5"); @@ -497,6 +507,7 @@ describe("the first backup of the account", () => { metadata: { title: "toast.mp4", fileType: "video" }, }); expect(toast).not.toHaveProperty("mlData"); + expect(toast).not.toHaveProperty("keywords"); expect(toast).not.toHaveProperty("imageMetadata"); expect(toast).not.toHaveProperty("imageMetadataError"); diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index c6258dd..f0c35cd 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -946,9 +946,19 @@ describe("the backup lock", () => { }); }); -// Also serves ML data: file 100 has `ML_PAYLOAD`, the other two have none. -// While `mlError` is set, every ML data request fails with it. -const ML_PAYLOAD = { face: { faces: [] }, clip: { embedding: [0.5, 0.25] } }; +// Also serves ML data: file 100 has `ML_PAYLOAD`, whose CLIP embedding is that +// of a photo of a sunset (see clip-keywords.test.ts), and the other two have +// none. While `mlError` is set, every ML data request fails with it. +const SUNSET_EMBEDDING: number[] = JSON.parse( + readFileSync( + new URL("../clip-keywords-sunset.json", import.meta.url), + "utf-8", + ), +).embedding; +const ML_PAYLOAD = { + face: { faces: [] }, + clip: { embedding: SUNSET_EMBEDDING }, +}; class MLClient extends MockClient { mlError?: string; @@ -968,16 +978,20 @@ const fileJSON = (outDir: string, fileID: number): Record => JSON.parse(readFileSync(saved(outDir, `${fileID}.json`), "utf-8")); describe("ML data in each file's JSON", () => { - it("writes a file's ML data, and no ML field for a file that has none", async () => { + // The JSON holds the ML data without its CLIP embedding, and in its place + // the keywords the embedding gives. + it("writes a file's ML data and keywords, and no ML field for a file that has none", async () => { const lib = await openLibrary(stubSource(), new MLClient()); const outDir = join(root, "backup"); const result = await lib.backup({ downloadDirectory: outDir }); expect(result.failed).toBe(0); - expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD); + expect(fileJSON(outDir, 100).mlData).toEqual({ face: { faces: [] } }); + expect(fileJSON(outDir, 100).keywords).toEqual(["Sunset"]); expect(fileJSON(outDir, 101)).not.toHaveProperty("mlData"); expect(fileJSON(outDir, 101)).not.toHaveProperty("mlDataError"); + expect(fileJSON(outDir, 101)).not.toHaveProperty("keywords"); await lib.close(); }); @@ -993,7 +1007,7 @@ describe("ML data in each file's JSON", () => { const result = await lib.backup({ downloadDirectory: outDir }); // File 100's ML data was cached by the first run and is kept. - expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD); + expect(fileJSON(outDir, 100).mlData).toEqual({ face: { faces: [] } }); expect(fileJSON(outDir, 100)).not.toHaveProperty("mlDataError"); for (const fileID of [101, 200]) { expect(fileJSON(outDir, fileID).mlDataError).toBe( @@ -1026,7 +1040,7 @@ describe("ML data in each file's JSON", () => { expect(result.failed).toBe(0); expect(result.skipped).toBe(3); - expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD); + expect(fileJSON(outDir, 100).mlData).toEqual({ face: { faces: [] } }); for (const fileID of [100, 101, 200]) { expect(fileJSON(outDir, fileID)).not.toHaveProperty("mlDataError"); } diff --git a/test/cli/metadata-backup.test.ts b/test/cli/metadata-backup.test.ts index 0b75804..d8d28e1 100644 --- a/test/cli/metadata-backup.test.ts +++ b/test/cli/metadata-backup.test.ts @@ -66,6 +66,15 @@ vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({ const TEST_EMAIL = "metabackup@example.com"; const TEST_PASSWORD = "metapass"; +// The CLIP embedding of a photo of a sunset (see clip-keywords.test.ts), which +// the ML data of beach.jpg holds. +const SUNSET_EMBEDDING: number[] = JSON.parse( + readFileSync( + new URL("../clip-keywords-sunset.json", import.meta.url), + "utf-8", + ), +).embedding; + let mock: MetaMockState; let testDir: string; @@ -230,7 +239,7 @@ const buildMetaMock = async (): Promise => { clip: { version: 1, client: "test", - embedding: [0.5, 0.6, 0.7], + embedding: SUNSET_EMBEDDING, }, }); const gzipped = gzipSync(Buffer.from(mlPayload)); @@ -432,7 +441,9 @@ describe("quak backup-metadata", () => { expect(account.email).toBe(TEST_EMAIL); }); - it("fetches and decrypts ML data by default", async () => { + // The CLIP embedding is left out of the ML data, and the keywords it gives + // are written in its place. + it("fetches and decrypts ML data by default, with keywords for the CLIP embedding", async () => { const outDir = join(testDir, "ml-data"); await runBackup(outDir); @@ -452,8 +463,8 @@ describe("quak backup-metadata", () => { expect(fileMeta.mlData.face.faces[0].faceID).toBe("face-abc"); expect(fileMeta.mlData.face.faces[0].score).toBeCloseTo(0.98); expect(fileMeta.mlData.face.faces[0].detection.box.x).toBeCloseTo(0.1); - expect(fileMeta.mlData.clip).toBeDefined(); - expect(fileMeta.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]); + expect(fileMeta.mlData).not.toHaveProperty("clip"); + expect(fileMeta.keywords).toEqual(["Sunset"]); }); it("extracts EXIF from downloaded files when --exif is set", async () => { @@ -541,7 +552,7 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => { "utf-8", ), ); - expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]); + expect(ok.keywords).toEqual(["Sunset"]); expect(ok.mlDataError).toBeUndefined(); const failed = JSON.parse( @@ -599,7 +610,7 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => { "utf-8", ), ); - expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]); + expect(ok.keywords).toEqual(["Sunset"]); }); it("exits 0 when every ML data request succeeds", async () => { diff --git a/test/clip-keywords-sunset.json b/test/clip-keywords-sunset.json new file mode 100644 index 0000000..499071f --- /dev/null +++ b/test/clip-keywords-sunset.json @@ -0,0 +1,109 @@ +{ + "photo": "https://upload.wikimedia.org/wikipedia/commons/d/db/Sunset_off_the_Massachusetts_coast_-_NOAA.jpg", + "sha256": "be62f3289c9bf7b75af68cd6978021d54e8a531d5d4cc493f6e9fbf5e1ab74d3", + "embedding": [ + -0.0021123015, -0.04437615, -0.062354308, 0.006249992, 0.022141185, + 0.011159019, 0.01227392, -0.00030532264, 0.03186437, 0.09196922, + 0.039111234, -0.037084445, -0.032962628, -0.019820206, -0.013141825, + 0.0047183544, 0.017137334, 0.046581306, 0.0047622235, 0.04982521, + 0.023128154, 0.032080866, 0.037709303, 0.019241406, 0.0032100384, + -0.014275843, 0.049337693, 0.025953151, 0.014263995, -0.0062328726, + -0.040504098, -0.015083303, -0.008123875, -0.006233639, -0.05352683, + -0.0048083956, 0.0763486, -0.013155067, 0.025114346, 0.022016674, + 0.0001588023, -0.010341572, -0.0037518279, -0.019181008, 0.04549337, + -0.0010725886, 0.004636866, 0.017177915, -0.0117317885, 0.0026097358, + 0.030738998, 0.013158572, -0.08856403, -0.0121391015, 0.011477229, + -0.0033104012, 0.035524756, -0.018607879, -0.0071739843, -0.031906743, + 0.0056713386, -0.0044195866, 0.00292471, 0.00061686034, -0.043075223, + 0.09040427, 0.001522537, -0.011445652, 0.009609886, -0.0053201183, + -0.0057452377, 0.00972853, -0.0024397348, -0.021255083, 0.015893811, + -0.006020548, -0.09428023, 0.014277583, -0.017975299, -0.011878315, + 0.011825521, 0.00014203975, -0.0020437292, -0.05305343, -0.024840152, + -0.011336872, -0.002796541, -0.00036171873, 0.004530499, 0.044486605, + 0.021602718, -0.01292857, 0.024530916, 0.019559948, -0.039928272, + -0.0681174, 0.049101, 0.02461532, 0.0365531, -0.04070358, -0.011011939, + -0.0027188526, -0.020370876, 0.012904856, 0.043157287, -0.051658224, + -0.3726012, 0.019549074, -0.0499252, -0.04800559, -0.011493859, + 0.02893256, 0.060341954, 0.0013929688, -0.051283866, -0.024286803, + 0.044124287, -0.005995806, -0.062817656, 0.007369614, -0.021470634, + -0.028147545, 0.00411659, 0.04234773, 0.03787647, 0.008579682, + -0.038568925, -0.027498025, -0.028416378, 0.0022012168, 0.010906191, + -0.013745787, 0.018267838, 0.013531097, -0.061559845, 0.004682517, + -0.005993303, 0.0028113858, -0.0028286583, 0.0026029332, -0.06693403, + -0.0072381413, -0.02419584, 0.0037009937, -0.0011296495, -0.030310456, + 0.0005999609, 0.009945354, 0.024915686, 0.038672045, 0.023009432, + -0.05093051, -0.09101409, -0.06686167, -0.06211481, -0.022600474, + 0.0024902367, -0.0067171026, -0.016769778, -0.0032803628, 0.008623083, + -0.0016110836, 0.014925073, 0.033811048, 0.012829449, -0.052575592, + 0.03728281, -0.0051201633, 0.021238064, -0.031086605, -0.0043402985, + 0.25054494, -0.028396713, -0.010725523, -0.023445267, 0.021751683, + 0.003130971, 0.012270548, -0.014836475, 0.014466885, 0.0021478906, + -0.088153616, 0.015356315, 0.039639276, -0.040283255, 0.014923924, + -0.029303031, -0.004475201, 0.047299225, -0.06726505, -0.012491676, + -0.002080582, 0.03506341, 0.04146718, 0.0074114986, -0.009582633, + 0.006623749, -0.05454898, -0.0034387247, -0.014780821, 0.03698328, + -0.034441486, 0.22032379, -0.021792699, 0.008001907, 0.0341019, + 0.0025784625, -0.0071689896, -0.014151225, -0.050547987, -0.030372301, + -0.03288388, -0.042987447, -0.0033656165, 0.021502906, -0.0030374376, + -0.024413, -0.03506603, 0.030891743, 0.0069994032, 0.004887366, + 0.030707076, 0.0065348223, 0.0008240555, 0.016809713, -0.024151647, + -0.0040458064, 0.044567548, 0.03908212, -0.061782435, 0.0026002831, + 0.02003242, -0.0115451, -0.045135964, -0.0663542, 0.005240975, + 0.0010777109, -0.009276002, -0.011919081, 0.027654119, -0.013634551, + 0.0129154315, -0.013195512, 0.040571272, -0.099290505, -0.04656693, + -0.20979854, 0.04181004, -0.06769166, -0.04121946, 0.018260444, + 0.005617506, -0.15694222, 0.032472793, 0.014189598, 0.008345103, + -0.041118417, 0.0074527003, -0.011494104, 0.004906474, -0.0027100458, + 0.0009994594, -0.041061398, -0.045274835, 0.015187793, 0.034275036, + -0.02813289, 0.02098019, -0.015043198, 0.03716667, 0.013542642, + -0.008104835, 0.014965486, -0.027978558, 0.021163663, -0.006942737, + -0.034969814, -0.038961593, 0.016802894, -0.012318175, -0.047821593, + -0.01933216, 0.0064432374, 0.0322358, -0.020437784, 0.02659447, + -0.03022022, -0.019443465, -0.094178624, -0.017009107, 0.03860671, + -0.00023695534, -0.033089854, -0.014677462, 0.012356399, 0.018040499, + -0.19360113, 0.011710817, -0.019871691, 0.020124774, 0.024948105, + -0.024573864, -0.028349072, -0.041731454, 0.032646254, 0.015098165, + 0.0018447012, -0.01129152, -0.02481687, 0.036048, 0.20797811, + 0.04317757, -0.017239755, 0.033431605, -0.011384422, 0.0099174455, + -0.19739223, -0.008901062, -0.056426458, 0.014591864, 0.0044474266, + 0.008209326, -0.012241819, 0.042553645, -0.021552453, 0.008611734, + 0.048944127, -0.013872049, 0.019179989, 0.008183264, -0.053223096, + -0.027327595, -0.0033542586, -0.012353858, -0.010941209, 0.010212546, + 0.029714134, -0.011385593, -0.0013529479, 0.014505385, 0.01148663, + 0.012613636, -0.0074430504, -0.014008101, -0.028146038, -0.042778917, + 0.02143165, 0.016812252, -0.06286215, 0.02085532, -0.035128277, + 0.0241282, 0.01397908, -0.034992814, 0.013153766, 0.0068236147, + 0.013594184, -0.06859393, 0.051849406, 0.029675119, -0.0037699796, + -0.00013216575, 0.02916647, 0.031859048, -0.015006051, 0.0001439966, + 0.0026584554, -0.0046948595, 0.039754897, 0.034873154, 0.0012870439, + -0.011516082, -0.074089095, 0.019241294, 0.00076195505, 0.0027334464, + -0.036206458, -0.0011109262, -0.048838835, -0.00204685, 0.01179991, + 0.0068136845, 0.009852495, 0.017308988, 0.03159545, 0.058803353, + 0.038116634, 0.008836143, -0.010460637, -0.010994099, -0.0012926942, + -0.048370842, 0.037696566, -0.08093239, -0.0041242186, -0.005866373, + 0.02048607, -0.12549639, 0.011705739, -0.0021425558, 0.018078592, + -0.038228087, 0.004245971, -0.11560507, 0.01381203, -0.02734893, + 0.029595908, 0.101082794, -0.042842254, -0.08397301, -0.1546237, + -0.037624564, 0.0069606556, -0.008020479, 0.0043268185, 0.016504597, + 0.044554908, -0.05240209, 0.020519335, -0.050994243, -0.019861093, + 0.022066997, -0.0188911, 0.012919696, -0.022872692, -0.012961374, + -0.020071112, 0.0074553066, 0.06402014, 0.04065211, 0.014139608, + 0.06221546, -0.02574805, 0.010114545, 0.020619567, 0.032264773, + -0.009148828, 0.022849955, -0.016593235, -0.032456633, -0.006737303, + -0.0037493357, -0.065912865, -0.0024181998, -0.0037048983, -0.019304488, + 0.009085037, 0.057529982, -0.0378338, 0.01792042, 0.01885012, + -0.0043600523, 0.017382368, -0.027792772, 0.006749118, -0.0053297766, + -0.011628587, 0.015736576, 0.024801968, 0.011543913, -0.01159873, + -0.000759976, 0.042183932, 0.05764386, -0.06422416, 0.010577374, + -0.02261249, 0.02282793, -0.05141075, -0.11287078, -0.020670144, + -0.020953743, 0.030885734, -0.012597732, -0.121472664, 0.028447151, + -0.0415664, -0.010061708, 0.06441338, 0.0271207, -0.0016693748, + -0.0045694243, 0.020342957, 0.030528478, 0.007129517, -0.0029600926, + -0.007061397, 0.06036486, -0.004736018, -0.0090786135, -0.07601116, + -0.039556272, -0.051269546, -0.019473685, 0.029858151, 0.028464556, + 0.012424476, 0.013742254, 0.008742927, 0.006065751, -0.009892663, + 0.07699999, -0.003443197, 0.0027863425, 0.029197995, 0.0211013, + 0.01098379, 0.016684106, -0.028917195, 0.008122037, 0.0265241, + -0.04461223 + ] +} diff --git a/test/clip-keywords.test.ts b/test/clip-keywords.test.ts new file mode 100644 index 0000000..1fb560a --- /dev/null +++ b/test/clip-keywords.test.ts @@ -0,0 +1,129 @@ +// `clipKeywords`: the keywords quak writes for a file in place of its CLIP +// embedding. +// +// Ente's ML data for a file holds a CLIP embedding, 512 numbers its image +// model gave the photo. Ente's photos app puts a file in a category of its +// Discover section when the cosine similarity of that embedding and the text +// embedding of the category's query is at least the category's minimum score, +// and keeps videos out of some categories. `clipKeywords` applies that rule +// and returns the titles of the categories a file falls in, highest score +// first; `quak backup` and `quak backup-metadata` write them as `keywords`. + +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +import { DISCOVER_CATEGORIES } from "../src/clip-keyword-vectors.js"; +import { clipKeywords } from "../src/clip-keywords.js"; + +// 512 numbers: the ones given, then zeros. +const vector = (...start: number[]): number[] => [ + ...start, + ...new Array(512 - start.length).fill(0), +]; + +const withEmbedding = (embedding: unknown) => ({ + face: { faces: [] }, + clip: { embedding }, +}); + +// Two made-up categories, each with a text embedding along one axis. The +// embedding [3, 4] has a cosine similarity of 3/5 with "Across" and 4/5 with +// "Up". Videos are kept out of "Across". +const ACROSS = { + title: "Across", + query: "across", + minScore: 0.6, + showVideo: false, + textEmbedding: vector(1), +}; +const UP = { + title: "Up", + query: "up", + minScore: 0.5, + showVideo: true, + textEmbedding: vector(0, 1), +}; + +describe("clipKeywords", () => { + it("gives each category whose minimum score the similarity reaches, highest score first", () => { + expect( + clipKeywords(withEmbedding(vector(3, 4)), false, [ACROSS, UP]), + ).toEqual(["Up", "Across"]); + }); + + it("leaves out a category when the similarity is just below its minimum score", () => { + const stricter = { ...ACROSS, minScore: 0.6 + 1e-9 }; + + expect( + clipKeywords(withEmbedding(vector(3, 4)), false, [stricter, UP]), + ).toEqual(["Up"]); + }); + + it("keeps a video out of a category that leaves out videos", () => { + expect( + clipKeywords(withEmbedding(vector(3, 4)), true, [ACROSS, UP]), + ).toEqual(["Up"]); + }); + + it("gives an empty list for an embedding that falls in no category", () => { + expect( + clipKeywords(withEmbedding(vector(0, 0, 1)), false, [ACROSS, UP]), + ).toEqual([]); + }); + + it.each([ + ["no CLIP embedding", { face: { faces: [] } }], + ["a CLIP part that is not an object", { clip: null }], + ["a CLIP part without an embedding", { clip: {} }], + ["an embedding of 511 numbers", withEmbedding(vector(3, 4).slice(1))], + ["an embedding with NaN in it", withEmbedding(vector(3, NaN))], + ["an embedding with Infinity in it", withEmbedding(vector(Infinity))], + ["an embedding of strings", withEmbedding(vector(3, 4).map(String))], + ["an embedding of zeros", withEmbedding(vector())], + ])("gives no keywords for ML data with %s", (_, mlData) => { + expect(clipKeywords(mlData, false, [ACROSS, UP])).toBeUndefined(); + }); +}); + +// The CLIP embedding Ente's image model gives a public-domain NOAA photo of a +// sunset, made by bin/clip-keyword-vectors from the photo at `photo`. +const SUNSET: { photo: string; sha256: string; embedding: number[] } = + JSON.parse( + readFileSync( + new URL("clip-keywords-sunset.json", import.meta.url), + "utf-8", + ), + ); + +describe("Ente's Discover categories", () => { + it("are the 16 of Ente's photos app, each with a text embedding of length 1", () => { + expect(DISCOVER_CATEGORIES.map((c) => c.title)).toEqual([ + "Identity", + "Screenshots", + "QR Codes", + "Receipts", + "Notes", + "Memes", + "Visiting Cards", + "Babies", + "Pets", + "Selfies", + "Wallpapers", + "Food", + "Celebrations", + "Sunset", + "Hills", + "Greenery", + ]); + for (const category of DISCOVER_CATEGORIES) { + expect(category.textEmbedding).toHaveLength(512); + expect(Math.hypot(...category.textEmbedding)).toBeCloseTo(1, 5); + } + }); + + it("give a photo of a sunset the keyword Sunset and no other", () => { + expect( + clipKeywords({ clip: { embedding: SUNSET.embedding } }, false), + ).toEqual(["Sunset"]); + }); +});