A file's JSON holds keywords in place of its CLIP embedding (closes #218)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
quak backup and quak backup-metadata write each file's ML data without its CLIP embedding, and keywords in its place: the titles of the Discover categories of Ente's photos app that the file falls in, by the app's rule. src/clip-keywords.ts applies the rule. src/clip-keyword-vectors.ts holds the 16 categories from Ente's discover.json with the text embedding Ente's CLIP text model gives each query. bin/clip-keyword-vectors writes that file and the test fixture in a digest-pinned Python image, with every download pinned by sha256; it runs by hand only. quak gains no dependency. Model: opus-5-5
This commit is contained in:
+14
-6
@@ -14,9 +14,9 @@
|
||||
// YYYY/YYYY-MM/YYYY-MM-DD/
|
||||
// YYYY-MM-DD.<fileID>.<ext> the decrypted bytes (the save path)
|
||||
// YYYY-MM-DD.<fileID>.json per-file metadata sidecar, with
|
||||
// the file's ML data and its
|
||||
// original's EXIF, XMP and
|
||||
// dimensions
|
||||
// the file's ML data, its
|
||||
// keywords, and its original's
|
||||
// EXIF, XMP and dimensions
|
||||
// collections/<name>/<title> symlink to the original
|
||||
// collections/<name>.json per-collection metadata
|
||||
// account.json the account's email and user ID
|
||||
@@ -77,6 +77,7 @@ import { readFile } from "node:fs/promises";
|
||||
import { dirname, extname, join, relative, resolve } from "node:path";
|
||||
import lockfile from "proper-lockfile";
|
||||
|
||||
import { clipKeywords } from "./clip-keywords.js";
|
||||
import { removeLeftoverTempFiles } from "./download/index.js";
|
||||
import { describeError } from "./errors.js";
|
||||
import {
|
||||
@@ -497,8 +498,9 @@ const imageMetadataFor = async (
|
||||
}
|
||||
};
|
||||
|
||||
// The file's JSON: its basic fields, its magic metadata, its ML data or the
|
||||
// reason the ML data is missing, and its image metadata.
|
||||
// The file's JSON: its basic fields, its magic metadata, its ML data without
|
||||
// the CLIP embedding and the keywords that embedding gives, or the reason the
|
||||
// ML data is missing, and its image metadata.
|
||||
const writeSidecar = (
|
||||
path: string,
|
||||
file: EnteFile,
|
||||
@@ -514,7 +516,13 @@ const writeSidecar = (
|
||||
};
|
||||
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
||||
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
||||
if (ml.mlData) meta.mlData = ml.mlData;
|
||||
if (ml.mlData) {
|
||||
const { clip: _clip, ...mlDataWithoutClip } = ml.mlData;
|
||||
meta.mlData = mlDataWithoutClip;
|
||||
const isVideo = file.metadata.fileType === "video";
|
||||
const keywords = clipKeywords(ml.mlData, isVideo);
|
||||
if (keywords) meta.keywords = keywords;
|
||||
}
|
||||
if (ml.mlDataError) meta.mlDataError = ml.mlDataError;
|
||||
if (image.imageMetadata) meta.imageMetadata = image.imageMetadata;
|
||||
if (image.imageMetadataError) {
|
||||
|
||||
Reference in New Issue
Block a user