Fetch, store, and index per-file ML data (closes #49)
check / check (push) Successful in 14s
check / check (push) Successful in 14s
Adds the machine-learning (magic) data layer: fetches per-file ML payloads (face detections + CLIP embeddings) via the existing metadata-backup fetch through the metadata pool after each refresh, decrypts and gunzips them, and stores one mldata/<fileID>.json per file by rename (present-means-complete). A derived index (mldata/clip.f32 + clip.json) loads in one read and is rebuilt whenever it disagrees with the payloads on disk in either direction, so an interrupted backfill self-heals. Never in metadata.json; incremental on later refreshes; progress via onProgress/status. Model: opus-4-8
This commit was merged in pull request #65.
This commit is contained in:
@@ -7,6 +7,7 @@ import {
|
||||
} from "./auth/login.js";
|
||||
import { unwrapAuth } from "./auth/unwrap.js";
|
||||
import { init, fromBase64, toBase64 } from "./crypto/index.js";
|
||||
import { fetchMLDataBatch, type MLData } from "./mldata-fetch.js";
|
||||
import { decryptCollection, decryptFile } from "./model/index.js";
|
||||
import {
|
||||
downloadFile as dlFile,
|
||||
@@ -273,6 +274,18 @@ export class Client {
|
||||
return files;
|
||||
}
|
||||
|
||||
// Fetch machine-learning data (face detections + CLIP embeddings) for up
|
||||
// to a batch of files, each decrypted with its own key. One request; the
|
||||
// library batches at `MLDATA_BATCH_SIZE` and schedules each batch through
|
||||
// its metadata request pool.
|
||||
async fetchMLData(args: {
|
||||
fileIDs: number[];
|
||||
fileKeys: Map<number, Uint8Array>;
|
||||
}): Promise<Map<number, MLData>> {
|
||||
this.assertLoggedIn();
|
||||
return fetchMLDataBatch(this.api, args.fileIDs, args.fileKeys);
|
||||
}
|
||||
|
||||
async downloadFile(
|
||||
file: EnteFile,
|
||||
outPath?: string,
|
||||
|
||||
Reference in New Issue
Block a user