Content-similarity search surface over the CLIP index (closes #50)
check / check (push) Failing after 23s

Adds lib.mldata search over the CLIP index (#49): forFile returns a file's stored payload; similar ranks nearest files by cosine on the CLIP embedding; searchByEmbedding ranks the index against a caller-supplied query vector. All RAM-only, reusing the packed Float32Array index and id list. No text encoder is bundled — the caller provides the query embedding.

Model: opus-4-8
This commit was merged in pull request #67.
This commit is contained in:
2026-09-22 18:14:13 +02:00
parent d7f415fe29
commit 224bd101ab
3 changed files with 240 additions and 4 deletions
+12 -4
View File
@@ -41,6 +41,7 @@ import {
type PhotosAPI,
type TimelineAPI,
} from "./read.js";
import { makeMLDataAPI, type MLDataAPI } from "./mlsearch.js";
export {
Album,
@@ -52,6 +53,7 @@ export {
type TimelineGroup,
type GroupBy,
} from "./read.js";
export { type MLDataAPI, type SimilarResult } from "./mlsearch.js";
import type { CollectionsPage, FilesPage } from "../client.js";
import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js";
import type { Collection, EnteFile } from "../model/types.js";
@@ -138,6 +140,10 @@ export class Library {
readonly albums: AlbumsAPI;
readonly photos: PhotosAPI;
readonly timeline: TimelineAPI;
// The content-similarity search surface over the CLIP index (issue #50).
// Present whether or not ML fetching is enabled; with no ML store it
// returns empty results.
readonly mldata: MLDataAPI;
private readonly client: LibraryClient;
private readonly store: MetadataStore;
@@ -146,7 +152,7 @@ export class Library {
private readonly onProgress?: RefreshProgressCallback;
private readonly pools: RequestPools;
// The ML-data cache, present only when the client can fetch ML data.
private readonly mldata?: MLDataStore;
private readonly mlStore?: MLDataStore;
private timer?: ReturnType<typeof setTimeout>;
private refreshing = false;
@@ -188,7 +194,7 @@ export class Library {
this.intervalMs = args.intervalMs;
this.onProgress = args.onProgress;
this.pools = args.pools;
this.mldata = args.mldata;
this.mlStore = args.mldata;
this.lastRecords = this.deriveNow();
// The read namespaces derive fresh from the store on each call, so they
@@ -197,6 +203,8 @@ export class Library {
this.albums = makeAlbumsAPI(derive);
this.photos = makePhotosAPI(derive);
this.timeline = makeTimelineAPI(derive);
// Reads the ML store live so results grow as ML data is fetched.
this.mldata = makeMLDataAPI(() => this.mlStore);
}
// Load the cache and start the refresh loop. With an empty cache the first
@@ -293,7 +301,7 @@ export class Library {
for (const c of collections) {
files += this.store.listFiles(c.id).length;
}
const ml = this.mldata?.stats();
const ml = this.mlStore?.stats();
return {
userID: this.store.userID,
collections: collections.length,
@@ -448,7 +456,7 @@ export class Library {
// advanced), through the metadata pool, and update the CLIP index. Guarded
// so passes never overlap; a failure is reported, not thrown.
private async runMLFetch(): Promise<void> {
const mldata = this.mldata;
const mldata = this.mlStore;
// Bind so the call keeps the client as its receiver when invoked
// through the pool below.
const fetchMLData = this.client.fetchMLData?.bind(this.client);