A file's JSON holds keywords in place of its CLIP embedding (closes #218)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
quak backup and quak backup-metadata write each file's ML data without its CLIP embedding, and keywords in its place: the titles of the Discover categories of Ente's photos app that the file falls in, by the app's rule. src/clip-keywords.ts applies the rule. src/clip-keyword-vectors.ts holds the 16 categories from Ente's discover.json with the text embedding Ente's CLIP text model gives each query. bin/clip-keyword-vectors writes that file and the test fixture in a digest-pinned Python image, with every download pinned by sha256; it runs by hand only. quak gains no dependency. Model: opus-5-5
This commit is contained in:
+12
-2
@@ -1,6 +1,7 @@
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import * as jpeg from "jpeg-js";
|
||||
import { clipKeywords } from "./clip-keywords.js";
|
||||
import type { Client } from "./client.js";
|
||||
import { describeError } from "./errors.js";
|
||||
import { readExifTags } from "./exif.js";
|
||||
@@ -97,7 +98,8 @@ const extractExif = async (
|
||||
|
||||
// Dump every decrypted metadata layer the account holds into a directory tree
|
||||
// of plain JSON: account, per-collection, and per-file records including the
|
||||
// private and public magic metadata and (by default) the ML data. Collections
|
||||
// private and public magic metadata and the ML data, with the keywords of its
|
||||
// CLIP embedding in place of the embedding (see clip-keywords.ts). Collections
|
||||
// and files are enumerated from the library's cache, which the caller refreshes
|
||||
// first. Returns how many files have no ML data because their ML data request
|
||||
// failed or their ML data did not decrypt, gunzip or parse; they are still
|
||||
@@ -230,8 +232,16 @@ export const runMetadataBackup = async (
|
||||
if (file.pubMagicMetadata)
|
||||
fileMeta.pubMagicMetadata = file.pubMagicMetadata;
|
||||
|
||||
// The ML data without the CLIP embedding, and the keywords that
|
||||
// embedding gives.
|
||||
const ml = mlDataMap.get(file.id);
|
||||
if (ml) fileMeta.mlData = ml;
|
||||
if (ml) {
|
||||
const { clip: _clip, ...mlDataWithoutClip } = ml;
|
||||
fileMeta.mlData = mlDataWithoutClip;
|
||||
const isVideo = file.metadata.fileType === "video";
|
||||
const keywords = clipKeywords(ml, isVideo);
|
||||
if (keywords) fileMeta.keywords = keywords;
|
||||
}
|
||||
const mlError = mlDataErrors.get(file.id);
|
||||
if (mlError) fileMeta.mlDataError = mlError;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user