README.md

@@ -791,10 +791,11 @@ as it was and goes into `failures.json` (see "Backup layout").
 
 `backup-metadata <dir>` writes `account.json` and, for each collection, a
 directory `collections/<id>-<name>/` holding `_collection.json` and a
-`<fileID>.json` for each of its files, with the file's ML data and keywords as
-`quak backup` writes them (see "Backup layout"). The collection's name has
-unsafe characters replaced and is cut short between two characters so that the
-directory's name fits in 255 bytes; the ID keeps the directories apart.
+`<fileID>.json` for each of its files. Each `<fileID>.json` holds the file's ML
+data and keywords as `quak backup` writes them (see "Backup layout"). The
+collection's name has unsafe characters replaced and is cut short between two
+characters so that the directory's name fits in 255 bytes; the ID keeps the
+directories apart.
 
 `backup-metadata` fetches ML data in requests of up to 200 files. When a request
 fails, the error is logged, each of its files is written with the reason in an
@@ -863,17 +864,38 @@ and its `magicMetadata` and `pubMagicMetadata` when it has them. Update times
 are in microseconds, as Ente records them.
 
 A file's JSON holds Ente's ML data for it as `mlData`, such as the faces Ente
-found, without its CLIP embedding. In place of the embedding it holds
-`keywords`, the titles of the categories of the Discover section of Ente's
-photos app that the file falls in, highest score first. The categories are
-Identity, Screenshots, QR Codes, Receipts, Notes, Memes, Visiting Cards, Babies,
-Pets, Selfies, Wallpapers, Food, Celebrations, Sunset, Hills and Greenery. As in
-Ente's app, a file falls in a category when the cosine similarity of its CLIP
-embedding and the text embedding of the category's query is at least the
-category's minimum score, and a video stays out of Identity, Screenshots, QR
-Codes, Visiting Cards and Wallpapers. A file with no ML data has neither field,
-and one whose ML data holds no usable CLIP embedding has no `keywords`.
-`backup-metadata` writes both the same way.
+found, without its CLIP embedding. It holds `keywords` in place of the
+embedding.
+
+The Discover section of Ente's photos app has 16 categories:
+
+- Identity (no videos)
+- Screenshots (no videos)
+- QR Codes (no videos)
+- Receipts
+- Notes
+- Memes
+- Visiting Cards (no videos)
+- Babies
+- Pets
+- Selfies
+- Wallpapers (no videos)
+- Food
+- Celebrations
+- Sunset
+- Hills
+- Greenery
+
+A file's keywords are the titles of the categories it falls in. A file's score
+for a category is the cosine similarity of its CLIP embedding and the text
+embedding of the category's query. As in Ente's app, the file falls in the
+category when that score is at least the category's minimum score. A video never
+falls in a category marked "no videos". The keywords are listed highest score
+first.
+
+A file with no ML data has neither `mlData` nor `keywords`. A file whose ML data
+holds no usable CLIP embedding has no `keywords`. `backup-metadata` writes both
+fields the same way.
 
 The backup waits for the library's ML data fetch to finish before it writes the
 JSON files. If that fetch fails, each file whose ML data is not in the cache
@@ -1289,18 +1311,18 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange`
   not already at its save path there as `photo.download()` does (and, with
   `includeThumbnails`, fetches thumbnails) through the content cache, waits for
   an ML data fetch, and rebuilds the on-disk backup tree with a durable failure
-  ledger. Each file's JSON holds its ML data and keywords and, for an image (for
-  a live photo, its image), its original's EXIF and XMP, with its dimensions for
-  a JPEG only; a video's JSON holds none of these three. A fetched original is
-  written straight to its save path and not into the cache, which then counts it
-  as present; one the cache already held is copied from there. `BackupOptions`:
-  `downloadDirectory` (falls back to the library's), `includeOriginals` (default
-  `true`), `includeThumbnails` (default `false`), `onlyAlbumNames`, `verify`
-  (default `false`), `onProgress`, and `lock`. See Backup layout above for the
-  tree it writes, and for the albums `BackupResult` lists in
-  `undecryptableAlbums` and `failedAlbums`. Whether the backup took the lock or
-  its caller did, once the lock is lost the backup rejects with
-  `lost the lock on <dir>: <reason>` before it starts another file.
+  ledger. Each file's JSON holds its ML data and keywords. An image's JSON also
+  holds the EXIF and XMP of its original, and its dimensions if it is a JPEG. A
+  live photo's JSON holds these for its image. A video's JSON holds none of
+  them. A fetched original is written straight to its save path and not into the
+  cache, which then counts it as present; one the cache already held is copied
+  from there. `BackupOptions`: `downloadDirectory` (falls back to the
+  library's), `includeOriginals` (default `true`), `includeThumbnails` (default
+  `false`), `onlyAlbumNames`, `verify` (default `false`), `onProgress`, and
+  `lock`. See Backup layout above for the tree it writes, and for the albums
+  `BackupResult` lists in `undecryptableAlbums` and `failedAlbums`. Whether the
+  backup took the lock or its caller did, once the lock is lost the backup
+  rejects with `lost the lock on <dir>: <reason>` before it starts another file.
 
 ### Request pools
 
@@ -1385,11 +1407,11 @@ against the upstream implementations in `web/packages/base/`,
 
 The keywords come from Ente too. `src/clip-keyword-vectors.ts` holds the
 Discover categories of Ente's photos app, from
-`mobile/apps/photos/assets/discover.json`, each with the text embedding Ente's
-CLIP text model gives its query. `bin/clip-keyword-vectors` writes that file,
-and the CLIP embedding of a public-domain photo that the tests use, from Ente's
-list and models; it needs docker, pins every download by sha256, and runs only
-by hand, followed by `make fmt`.
+`mobile/apps/photos/assets/discover.json`, each with its query's text embedding
+from Ente's CLIP text model. `bin/clip-keyword-vectors` writes that file and the
+CLIP embedding of a public-domain photo that the tests use. It runs Ente's
+models in Docker, with every download pinned by sha256. It is run by hand only,
+followed by `make fmt`.
 
 ## For LLMs
 

bin/clip-keyword-vectors

@@ -5,21 +5,26 @@
 # no gate runs it. It downloads about 215 MB, most of it Ente's two CLIP models.
 set -eu
 
-cd "$(dirname "$0")/.."
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 
 # python:3.13-slim (Python 3.13.17 on Debian 13.7), 2026-10-21. The wheels
 # below are the ones for this image on linux/amd64.
-image=python@sha256:57b29a7c050c4e679e4bea572e14b3965cf4b5839de2da49d55e49ffc4024f39
+IMAGE=python@sha256:57b29a7c050c4e679e4bea572e14b3965cf4b5839de2da49d55e49ffc4024f39
 
-docker run --rm -i --platform linux/amd64 \
-    --user "$(id -u):$(id -g)" -e HOME=/tmp \
-    -v "$PWD:/repo" -w /repo "$image" sh -c '
-        python -m venv /tmp/venv &&
-        /tmp/venv/bin/pip install --quiet --disable-pip-version-check \
-            --no-cache-dir --no-deps --require-hashes -r /dev/stdin &&
-        /tmp/venv/bin/python bin/clip-keyword-vectors.py' <<'EOF'
+main() {
+    cd "$ROOT"
+    docker run --rm -i --platform linux/amd64 \
+        --user "$(id -u):$(id -g)" -e HOME=/tmp \
+        -v "$ROOT:/repo" -w /repo "$IMAGE" sh -c '
+            python -m venv /tmp/venv &&
+            /tmp/venv/bin/pip install --quiet --disable-pip-version-check \
+                --no-cache-dir --no-deps --require-hashes -r /dev/stdin &&
+            /tmp/venv/bin/python bin/clip-keyword-vectors.py' <<'EOF'
 numpy==2.5.3 --hash=sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321
 onnxruntime==1.30.0 --hash=sha256:86f940afc801ea9681a4da8af84fbe95e1d9ea7d80903952cc1bfad54faad38f
 pillow==12.3.0 --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3
 tokenizers==0.23.2 --hash=sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305
 EOF
+}
+
+main "$@"

src/clip-keywords.ts

@@ -10,6 +10,7 @@
 // their text embeddings are in `clip-keyword-vectors.ts`.
 
 import { DISCOVER_CATEGORIES } from "./clip-keyword-vectors.js";
+import { clipEmbedding } from "./library/mldata.js";
 import type { MLData } from "./mldata-fetch.js";
 
 // How many numbers a CLIP embedding from Ente's image model has.
@@ -33,13 +34,10 @@ export const clipKeywords = (
     isVideo: boolean,
     categories = DISCOVER_CATEGORIES,
 ): string[] | undefined => {
-    const clip = mlData.clip;
-    if (typeof clip !== "object" || clip === null) return undefined;
-    const embedding = (clip as { embedding?: unknown }).embedding;
+    const embedding = clipEmbedding(mlData);
     if (
-        !Array.isArray(embedding) ||
+        !embedding ||
         embedding.length !== EMBEDDING_LENGTH ||
-        !embedding.every((v) => typeof v === "number" && Number.isFinite(v)) ||
         length(embedding) === 0
     ) {
         return undefined;

src/library/mldata.ts

@@ -61,7 +61,7 @@ export interface MLIndex {
 // Pull the CLIP embedding out of a payload, or undefined when it is absent or
 // misshapen. Kept strict so a bad payload is skipped rather than corrupting the
 // packed index.
-const clipEmbedding = (payload: MLData): number[] | undefined => {
+export const clipEmbedding = (payload: MLData): number[] | undefined => {
     const clip = payload.clip;
     if (typeof clip !== "object" || clip === null) return undefined;
     const embedding = (clip as { embedding?: unknown }).embedding;