sneak/quak
1
0
Fork 0
Code Issues 3 Pull Requests 1 Actions 9 Packages Projects Releases Wiki Activity

A file's JSON holds keywords in place of its CLIP embedding (closes #218) #220

Merged
clawbot merged 6 commits from issue-218-clip-keywords into next 2026-10-22 23:32:25 +02:00
Conversation 9 Commits 6 Files Changed 14
+290 -16
5 changed files with 290 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 4863833f0a - Show all commits
test/cli/backup-end-to-end.test.ts
+14 -3
View File
@@ -86,10 +86,17 @@ const DAY = "2024/2024-03/2024-03-01";
const TOAST = new TextEncoder().encode("a few seconds of the wedding toast");
// Ente's ML data for beach.jpg: a face it found, and the photo's CLIP
// embedding.
// embedding, here that of a photo of a sunset (see clip-keywords.test.ts).
const ML_DATA = {
face: { faces: [{ faceID: "100_1", score: 0.98 }] },
clip: { embedding: [0.5, 0.6, 0.7] },
clip: {
embedding: JSON.parse(
readFileSync(
new URL("../clip-keywords-sunset.json", import.meta.url),
"utf-8",
),
).embedding,
},
};
// Each file's ID, its owner, its original as uploaded, and the metadata its
@@ -486,7 +493,10 @@ describe("the first backup of the account", () => {
},
updationTime: TAKEN,
});
expect(beach.mlData).toEqual(ML_DATA);
// The JSON holds the ML data without its CLIP embedding, and the
// keywords the embedding gives in its place.
expect(beach.mlData).toEqual({ face: ML_DATA.face });
expect(beach.keywords).toEqual(["Sunset"]);
expect(beach.imageMetadata.exif.Make.description).toBe("Canon");
expect(beach.imageMetadata.exif.Model.description).toBe("EOS R5");
@@ -497,6 +507,7 @@ describe("the first backup of the account", () => {
metadata: { title: "toast.mp4", fileType: "video" },
});
expect(toast).not.toHaveProperty("mlData");
expect(toast).not.toHaveProperty("keywords");
expect(toast).not.toHaveProperty("imageMetadata");
expect(toast).not.toHaveProperty("imageMetadataError");
test/cli/backup.test.ts
+21 -7
View File
@@ -946,9 +946,19 @@ describe("the backup lock", () => {
});
});
// Also serves ML data: file 100 has `ML_PAYLOAD`, the other two have none.
// While `mlError` is set, every ML data request fails with it.
const ML_PAYLOAD = { face: { faces: [] }, clip: { embedding: [0.5, 0.25] } };
// Also serves ML data: file 100 has `ML_PAYLOAD`, whose CLIP embedding is that
// of a photo of a sunset (see clip-keywords.test.ts), and the other two have
// none. While `mlError` is set, every ML data request fails with it.
const SUNSET_EMBEDDING: number[] = JSON.parse(
readFileSync(
new URL("../clip-keywords-sunset.json", import.meta.url),
"utf-8",
),
).embedding;
const ML_PAYLOAD = {
face: { faces: [] },
clip: { embedding: SUNSET_EMBEDDING },
};
class MLClient extends MockClient {
mlError?: string;
@@ -968,16 +978,20 @@ const fileJSON = (outDir: string, fileID: number): Record<string, unknown> =>
JSON.parse(readFileSync(saved(outDir, `${fileID}.json`), "utf-8"));
describe("ML data in each file's JSON", () => {
it("writes a file's ML data, and no ML field for a file that has none", async () => {
// The JSON holds the ML data without its CLIP embedding, and in its place
// the keywords the embedding gives.
it("writes a file's ML data and keywords, and no ML field for a file that has none", async () => {
const lib = await openLibrary(stubSource(), new MLClient());
const outDir = join(root, "backup");
const result = await lib.backup({ downloadDirectory: outDir });
expect(result.failed).toBe(0);
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
expect(fileJSON(outDir, 100).mlData).toEqual({ face: { faces: [] } });
expect(fileJSON(outDir, 100).keywords).toEqual(["Sunset"]);
expect(fileJSON(outDir, 101)).not.toHaveProperty("mlData");
expect(fileJSON(outDir, 101)).not.toHaveProperty("mlDataError");
expect(fileJSON(outDir, 101)).not.toHaveProperty("keywords");
await lib.close();
});
@@ -993,7 +1007,7 @@ describe("ML data in each file's JSON", () => {
const result = await lib.backup({ downloadDirectory: outDir });
// File 100's ML data was cached by the first run and is kept.
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
expect(fileJSON(outDir, 100).mlData).toEqual({ face: { faces: [] } });
expect(fileJSON(outDir, 100)).not.toHaveProperty("mlDataError");
for (const fileID of [101, 200]) {
expect(fileJSON(outDir, fileID).mlDataError).toBe(
@@ -1026,7 +1040,7 @@ describe("ML data in each file's JSON", () => {
expect(result.failed).toBe(0);
expect(result.skipped).toBe(3);
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
expect(fileJSON(outDir, 100).mlData).toEqual({ face: { faces: [] } });
for (const fileID of [100, 101, 200]) {
expect(fileJSON(outDir, fileID)).not.toHaveProperty("mlDataError");
}
test/cli/metadata-backup.test.ts
+17 -6
View File
@@ -66,6 +66,15 @@ vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({
const TEST_EMAIL = "metabackup@example.com";
const TEST_PASSWORD = "metapass";
// The CLIP embedding of a photo of a sunset (see clip-keywords.test.ts), which
// the ML data of beach.jpg holds.
const SUNSET_EMBEDDING: number[] = JSON.parse(
readFileSync(
new URL("../clip-keywords-sunset.json", import.meta.url),
"utf-8",
),
).embedding;
let mock: MetaMockState;
let testDir: string;
@@ -230,7 +239,7 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
clip: {
version: 1,
client: "test",
embedding: [0.5, 0.6, 0.7],
embedding: SUNSET_EMBEDDING,
},
});
const gzipped = gzipSync(Buffer.from(mlPayload));
@@ -432,7 +441,9 @@ describe("quak backup-metadata", () => {
expect(account.email).toBe(TEST_EMAIL);
});
it("fetches and decrypts ML data by default", async () => {
// The CLIP embedding is left out of the ML data, and the keywords it gives
// are written in its place.
it("fetches and decrypts ML data by default, with keywords for the CLIP embedding", async () => {
const outDir = join(testDir, "ml-data");
await runBackup(outDir);
@@ -452,8 +463,8 @@ describe("quak backup-metadata", () => {
expect(fileMeta.mlData.face.faces[0].faceID).toBe("face-abc");
expect(fileMeta.mlData.face.faces[0].score).toBeCloseTo(0.98);
expect(fileMeta.mlData.face.faces[0].detection.box.x).toBeCloseTo(0.1);
expect(fileMeta.mlData.clip).toBeDefined();
expect(fileMeta.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
expect(fileMeta.mlData).not.toHaveProperty("clip");
expect(fileMeta.keywords).toEqual(["Sunset"]);
});
it("extracts EXIF from downloaded files when --exif is set", async () => {
@@ -541,7 +552,7 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => {
"utf-8",
),
);
expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
expect(ok.keywords).toEqual(["Sunset"]);
expect(ok.mlDataError).toBeUndefined();
const failed = JSON.parse(
@@ -599,7 +610,7 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => {
"utf-8",
),
);
expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
expect(ok.keywords).toEqual(["Sunset"]);
});
it("exits 0 when every ML data request succeeds", async () => {
test/clip-keywords-sunset.json
+109
View File
@@ -0,0 +1,109 @@
{
"photo": "https://upload.wikimedia.org/wikipedia/commons/d/db/Sunset_off_the_Massachusetts_coast_-_NOAA.jpg",
"sha256": "be62f3289c9bf7b75af68cd6978021d54e8a531d5d4cc493f6e9fbf5e1ab74d3",
"embedding": [
-0.0021123015, -0.04437615, -0.062354308, 0.006249992, 0.022141185,
0.011159019, 0.01227392, -0.00030532264, 0.03186437, 0.09196922,
0.039111234, -0.037084445, -0.032962628, -0.019820206, -0.013141825,
0.0047183544, 0.017137334, 0.046581306, 0.0047622235, 0.04982521,
0.023128154, 0.032080866, 0.037709303, 0.019241406, 0.0032100384,
-0.014275843, 0.049337693, 0.025953151, 0.014263995, -0.0062328726,
-0.040504098, -0.015083303, -0.008123875, -0.006233639, -0.05352683,
-0.0048083956, 0.0763486, -0.013155067, 0.025114346, 0.022016674,
0.0001588023, -0.010341572, -0.0037518279, -0.019181008, 0.04549337,
-0.0010725886, 0.004636866, 0.017177915, -0.0117317885, 0.0026097358,
0.030738998, 0.013158572, -0.08856403, -0.0121391015, 0.011477229,
-0.0033104012, 0.035524756, -0.018607879, -0.0071739843, -0.031906743,
0.0056713386, -0.0044195866, 0.00292471, 0.00061686034, -0.043075223,
0.09040427, 0.001522537, -0.011445652, 0.009609886, -0.0053201183,
-0.0057452377, 0.00972853, -0.0024397348, -0.021255083, 0.015893811,
-0.006020548, -0.09428023, 0.014277583, -0.017975299, -0.011878315,
0.011825521, 0.00014203975, -0.0020437292, -0.05305343, -0.024840152,
-0.011336872, -0.002796541, -0.00036171873, 0.004530499, 0.044486605,
0.021602718, -0.01292857, 0.024530916, 0.019559948, -0.039928272,
-0.0681174, 0.049101, 0.02461532, 0.0365531, -0.04070358, -0.011011939,
-0.0027188526, -0.020370876, 0.012904856, 0.043157287, -0.051658224,
-0.3726012, 0.019549074, -0.0499252, -0.04800559, -0.011493859,
0.02893256, 0.060341954, 0.0013929688, -0.051283866, -0.024286803,
0.044124287, -0.005995806, -0.062817656, 0.007369614, -0.021470634,
-0.028147545, 0.00411659, 0.04234773, 0.03787647, 0.008579682,
-0.038568925, -0.027498025, -0.028416378, 0.0022012168, 0.010906191,
-0.013745787, 0.018267838, 0.013531097, -0.061559845, 0.004682517,
-0.005993303, 0.0028113858, -0.0028286583, 0.0026029332, -0.06693403,
-0.0072381413, -0.02419584, 0.0037009937, -0.0011296495, -0.030310456,
0.0005999609, 0.009945354, 0.024915686, 0.038672045, 0.023009432,
-0.05093051, -0.09101409, -0.06686167, -0.06211481, -0.022600474,
0.0024902367, -0.0067171026, -0.016769778, -0.0032803628, 0.008623083,
-0.0016110836, 0.014925073, 0.033811048, 0.012829449, -0.052575592,
0.03728281, -0.0051201633, 0.021238064, -0.031086605, -0.0043402985,
0.25054494, -0.028396713, -0.010725523, -0.023445267, 0.021751683,
0.003130971, 0.012270548, -0.014836475, 0.014466885, 0.0021478906,
-0.088153616, 0.015356315, 0.039639276, -0.040283255, 0.014923924,
-0.029303031, -0.004475201, 0.047299225, -0.06726505, -0.012491676,
-0.002080582, 0.03506341, 0.04146718, 0.0074114986, -0.009582633,
0.006623749, -0.05454898, -0.0034387247, -0.014780821, 0.03698328,
-0.034441486, 0.22032379, -0.021792699, 0.008001907, 0.0341019,
0.0025784625, -0.0071689896, -0.014151225, -0.050547987, -0.030372301,
-0.03288388, -0.042987447, -0.0033656165, 0.021502906, -0.0030374376,
-0.024413, -0.03506603, 0.030891743, 0.0069994032, 0.004887366,
0.030707076, 0.0065348223, 0.0008240555, 0.016809713, -0.024151647,
-0.0040458064, 0.044567548, 0.03908212, -0.061782435, 0.0026002831,
0.02003242, -0.0115451, -0.045135964, -0.0663542, 0.005240975,
0.0010777109, -0.009276002, -0.011919081, 0.027654119, -0.013634551,
0.0129154315, -0.013195512, 0.040571272, -0.099290505, -0.04656693,
-0.20979854, 0.04181004, -0.06769166, -0.04121946, 0.018260444,
0.005617506, -0.15694222, 0.032472793, 0.014189598, 0.008345103,
-0.041118417, 0.0074527003, -0.011494104, 0.004906474, -0.0027100458,
0.0009994594, -0.041061398, -0.045274835, 0.015187793, 0.034275036,
-0.02813289, 0.02098019, -0.015043198, 0.03716667, 0.013542642,
-0.008104835, 0.014965486, -0.027978558, 0.021163663, -0.006942737,
-0.034969814, -0.038961593, 0.016802894, -0.012318175, -0.047821593,
-0.01933216, 0.0064432374, 0.0322358, -0.020437784, 0.02659447,
-0.03022022, -0.019443465, -0.094178624, -0.017009107, 0.03860671,
-0.00023695534, -0.033089854, -0.014677462, 0.012356399, 0.018040499,
-0.19360113, 0.011710817, -0.019871691, 0.020124774, 0.024948105,
-0.024573864, -0.028349072, -0.041731454, 0.032646254, 0.015098165,
0.0018447012, -0.01129152, -0.02481687, 0.036048, 0.20797811,
0.04317757, -0.017239755, 0.033431605, -0.011384422, 0.0099174455,
-0.19739223, -0.008901062, -0.056426458, 0.014591864, 0.0044474266,
0.008209326, -0.012241819, 0.042553645, -0.021552453, 0.008611734,
0.048944127, -0.013872049, 0.019179989, 0.008183264, -0.053223096,
-0.027327595, -0.0033542586, -0.012353858, -0.010941209, 0.010212546,
0.029714134, -0.011385593, -0.0013529479, 0.014505385, 0.01148663,
0.012613636, -0.0074430504, -0.014008101, -0.028146038, -0.042778917,
0.02143165, 0.016812252, -0.06286215, 0.02085532, -0.035128277,
0.0241282, 0.01397908, -0.034992814, 0.013153766, 0.0068236147,
0.013594184, -0.06859393, 0.051849406, 0.029675119, -0.0037699796,
-0.00013216575, 0.02916647, 0.031859048, -0.015006051, 0.0001439966,
0.0026584554, -0.0046948595, 0.039754897, 0.034873154, 0.0012870439,
-0.011516082, -0.074089095, 0.019241294, 0.00076195505, 0.0027334464,
-0.036206458, -0.0011109262, -0.048838835, -0.00204685, 0.01179991,
0.0068136845, 0.009852495, 0.017308988, 0.03159545, 0.058803353,
0.038116634, 0.008836143, -0.010460637, -0.010994099, -0.0012926942,
-0.048370842, 0.037696566, -0.08093239, -0.0041242186, -0.005866373,
0.02048607, -0.12549639, 0.011705739, -0.0021425558, 0.018078592,
-0.038228087, 0.004245971, -0.11560507, 0.01381203, -0.02734893,
0.029595908, 0.101082794, -0.042842254, -0.08397301, -0.1546237,
-0.037624564, 0.0069606556, -0.008020479, 0.0043268185, 0.016504597,
0.044554908, -0.05240209, 0.020519335, -0.050994243, -0.019861093,
0.022066997, -0.0188911, 0.012919696, -0.022872692, -0.012961374,
-0.020071112, 0.0074553066, 0.06402014, 0.04065211, 0.014139608,
0.06221546, -0.02574805, 0.010114545, 0.020619567, 0.032264773,
-0.009148828, 0.022849955, -0.016593235, -0.032456633, -0.006737303,
-0.0037493357, -0.065912865, -0.0024181998, -0.0037048983, -0.019304488,
0.009085037, 0.057529982, -0.0378338, 0.01792042, 0.01885012,
-0.0043600523, 0.017382368, -0.027792772, 0.006749118, -0.0053297766,
-0.011628587, 0.015736576, 0.024801968, 0.011543913, -0.01159873,
-0.000759976, 0.042183932, 0.05764386, -0.06422416, 0.010577374,
-0.02261249, 0.02282793, -0.05141075, -0.11287078, -0.020670144,
-0.020953743, 0.030885734, -0.012597732, -0.121472664, 0.028447151,
-0.0415664, -0.010061708, 0.06441338, 0.0271207, -0.0016693748,
-0.0045694243, 0.020342957, 0.030528478, 0.007129517, -0.0029600926,
-0.007061397, 0.06036486, -0.004736018, -0.0090786135, -0.07601116,
-0.039556272, -0.051269546, -0.019473685, 0.029858151, 0.028464556,
0.012424476, 0.013742254, 0.008742927, 0.006065751, -0.009892663,
0.07699999, -0.003443197, 0.0027863425, 0.029197995, 0.0211013,
0.01098379, 0.016684106, -0.028917195, 0.008122037, 0.0265241,
-0.04461223
]
}
test/clip-keywords.test.ts
+129
View File
@@ -0,0 +1,129 @@
// `clipKeywords`: the keywords quak writes for a file in place of its CLIP
// embedding.
//
// Ente's ML data for a file holds a CLIP embedding, 512 numbers its image
// model gave the photo. Ente's photos app puts a file in a category of its
// Discover section when the cosine similarity of that embedding and the text
// embedding of the category's query is at least the category's minimum score,
// and keeps videos out of some categories. `clipKeywords` applies that rule
// and returns the titles of the categories a file falls in, highest score
// first; `quak backup` and `quak backup-metadata` write them as `keywords`.
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
import { DISCOVER_CATEGORIES } from "../src/clip-keyword-vectors.js";
import { clipKeywords } from "../src/clip-keywords.js";
// 512 numbers: the ones given, then zeros.
const vector = (...start: number[]): number[] => [
...start,
...new Array<number>(512 - start.length).fill(0),
];
const withEmbedding = (embedding: unknown) => ({
face: { faces: [] },
clip: { embedding },
});
// Two made-up categories, each with a text embedding along one axis. The
// embedding [3, 4] has a cosine similarity of 3/5 with "Across" and 4/5 with
// "Up". Videos are kept out of "Across".
const ACROSS = {
title: "Across",
query: "across",
minScore: 0.6,
showVideo: false,
textEmbedding: vector(1),
};
const UP = {
title: "Up",
query: "up",
minScore: 0.5,
showVideo: true,
textEmbedding: vector(0, 1),
};
describe("clipKeywords", () => {
it("gives each category whose minimum score the similarity reaches, highest score first", () => {
expect(
clipKeywords(withEmbedding(vector(3, 4)), false, [ACROSS, UP]),
).toEqual(["Up", "Across"]);
});
it("leaves out a category when the similarity is just below its minimum score", () => {
const stricter = { ...ACROSS, minScore: 0.6 + 1e-9 };
expect(
clipKeywords(withEmbedding(vector(3, 4)), false, [stricter, UP]),
).toEqual(["Up"]);
});
it("keeps a video out of a category that leaves out videos", () => {
expect(
clipKeywords(withEmbedding(vector(3, 4)), true, [ACROSS, UP]),
).toEqual(["Up"]);
});
it("gives an empty list for an embedding that falls in no category", () => {
expect(
clipKeywords(withEmbedding(vector(0, 0, 1)), false, [ACROSS, UP]),
).toEqual([]);
});
it.each([
["no CLIP embedding", { face: { faces: [] } }],
["a CLIP part that is not an object", { clip: null }],
["a CLIP part without an embedding", { clip: {} }],
["an embedding of 511 numbers", withEmbedding(vector(3, 4).slice(1))],
["an embedding with NaN in it", withEmbedding(vector(3, NaN))],
["an embedding with Infinity in it", withEmbedding(vector(Infinity))],
["an embedding of strings", withEmbedding(vector(3, 4).map(String))],
["an embedding of zeros", withEmbedding(vector())],
])("gives no keywords for ML data with %s", (_, mlData) => {
expect(clipKeywords(mlData, false, [ACROSS, UP])).toBeUndefined();
});
});
// The CLIP embedding Ente's image model gives a public-domain NOAA photo of a
// sunset, made by bin/clip-keyword-vectors from the photo at `photo`.
const SUNSET: { photo: string; sha256: string; embedding: number[] } =
JSON.parse(
readFileSync(
new URL("clip-keywords-sunset.json", import.meta.url),
"utf-8",
),
);
describe("Ente's Discover categories", () => {
it("are the 16 of Ente's photos app, each with a text embedding of length 1", () => {
expect(DISCOVER_CATEGORIES.map((c) => c.title)).toEqual([
"Identity",
"Screenshots",
"QR Codes",
"Receipts",
"Notes",
"Memes",
"Visiting Cards",
"Babies",
"Pets",
"Selfies",
"Wallpapers",
"Food",
"Celebrations",
"Sunset",
"Hills",
"Greenery",
]);
for (const category of DISCOVER_CATEGORIES) {
expect(category.textEmbedding).toHaveLength(512);
expect(Math.hypot(...category.textEmbedding)).toBeCloseTo(1, 5);
}
});
it("give a photo of a sunset the keyword Sunset and no other", () => {
expect(
clipKeywords({ clip: { embedding: SUNSET.embedding } }, false),
).toEqual(["Sunset"]);
});
});