A file's JSON holds keywords in place of its CLIP embedding (closes #218)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
`quak backup` and `quak backup-metadata` write each file's ML data without its CLIP embedding, and `keywords` in its place: the titles of the Discover categories of Ente's photos app that the file falls in, highest score first. A file falls in a category when the cosine similarity of its CLIP embedding and the text embedding of the category's query reaches the category's minimum score; as in Ente's app, five categories leave out videos. The 16 text embeddings ship in `src/clip-keyword-vectors.ts`, written by `bin/clip-keyword-vectors` from Ente's list and CLIP text model. quak runs no model and gains no dependency. Disclosure: the text embeddings are committed model output, against REPO_POLICIES' rule on generated files. Model: opus-5-5
This commit was merged in pull request #220.
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
// Keywords for a file from its CLIP embedding.
|
||||
//
|
||||
// Ente stores no keywords. A file's ML data holds a CLIP embedding, the 512
|
||||
// numbers Ente's image model gave the photo, and Ente's photos app sorts files
|
||||
// into the categories of its Discover section with it: a file falls in a
|
||||
// category when the cosine similarity of its CLIP embedding and the text
|
||||
// embedding of the category's query is at least the category's `minScore`,
|
||||
// unless the file is a video and the category leaves out videos. A file's
|
||||
// keywords are the titles of the categories it falls in. The categories and
|
||||
// their text embeddings are in `clip-keyword-vectors.ts`.
|
||||
|
||||
import { DISCOVER_CATEGORIES } from "./clip-keyword-vectors.js";
|
||||
import { clipEmbedding } from "./library/mldata.js";
|
||||
import type { MLData } from "./mldata-fetch.js";
|
||||
|
||||
// How many numbers a CLIP embedding from Ente's image model has.
|
||||
const EMBEDDING_LENGTH = 512;
|
||||
|
||||
const norm = (vector: number[]): number => Math.hypot(...vector);
|
||||
|
||||
const cosineSimilarity = (a: number[], b: number[]): number => {
|
||||
let dot = 0;
|
||||
a.forEach((value, i) => {
|
||||
dot += value * (b[i] ?? 0);
|
||||
});
|
||||
return dot / (norm(a) * norm(b));
|
||||
};
|
||||
|
||||
// The titles of the Discover categories a file falls in, highest score first,
|
||||
// or undefined when its ML data has no CLIP embedding of 512 finite numbers
|
||||
// that are not all zero. Tests pass their own `categories`.
|
||||
export const clipKeywords = (
|
||||
mlData: MLData,
|
||||
isVideo: boolean,
|
||||
categories = DISCOVER_CATEGORIES,
|
||||
): string[] | undefined => {
|
||||
const embedding = clipEmbedding(mlData);
|
||||
if (
|
||||
!embedding ||
|
||||
embedding.length !== EMBEDDING_LENGTH ||
|
||||
norm(embedding) === 0
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const matches: { title: string; score: number }[] = [];
|
||||
for (const category of categories) {
|
||||
if (isVideo && !category.showVideo) continue;
|
||||
const score = cosineSimilarity(embedding, category.textEmbedding);
|
||||
if (score >= category.minScore) {
|
||||
matches.push({ title: category.title, score });
|
||||
}
|
||||
}
|
||||
return matches
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.map((match) => match.title);
|
||||
};
|
||||
Reference in New Issue
Block a user