Fetch, store, and index per-file ML data (closes #49)

Adds the machine-learning (magic) data layer: fetches per-file ML payloads (face detections + CLIP embeddings) via the existing metadata-backup fetch through the metadata pool after each refresh, decrypts and gunzips them, and stores one mldata/<fileID>.json per file by rename (present-means-complete). A derived index (mldata/clip.f32 + clip.json) loads in one read and is rebuilt whenever it disagrees with the payloads on disk in either direction, so an interrupted backfill self-heals. Never in metadata.json; incremental on later refreshes; progress via onProgress/status.

Model: opus-4-8

src/metadata-backup.ts

@@ -1,4 +1,3 @@
-import { gunzipSync } from "node:zlib";
 import {
     mkdirSync,
     mkdtempSync,
@@ -11,7 +10,7 @@ import { tmpdir } from "node:os";
 import * as jpeg from "jpeg-js";
 import exifReader from "exif-reader";
 import type { Client } from "./client.js";
-import { decryptBlob, fromBase64 } from "./crypto/index.js";
+import { fetchMLData } from "./mldata-fetch.js";
 import type { EnteFile } from "./model/types.js";
 
 export type ProgressCallback = (message: string) => void;
@@ -24,50 +23,6 @@ export interface MetadataBackupOptions {
 const sanitizePath = (name: string): string =>
     name.replace(/[/\\:*?"<>|]/g, "_").replace(/^\.+/, "_");
 
-interface RawRemoteFileData {
-    fileID: number;
-    encryptedData: string;
-    decryptionHeader: string;
-    updatedAt?: number;
-}
-
-const fetchMLDataForFiles = async (
-    client: Client,
-    fileIDs: number[],
-    fileKeys: Map<number, Uint8Array>,
-): Promise<Map<number, Record<string, unknown>>> => {
-    const api = client.getApiClient();
-    const result = new Map<number, Record<string, unknown>>();
-    const batchSize = 200;
-
-    for (let i = 0; i < fileIDs.length; i += batchSize) {
-        const batch = fileIDs.slice(i, i + batchSize);
-        const { data } = await api.postJSON<{ data: RawRemoteFileData[] }>(
-            "/files/data/fetch",
-            { type: "mldata", fileIDs: batch },
-        );
-
-        for (const entry of data ?? []) {
-            const key = fileKeys.get(entry.fileID);
-            if (!key) continue;
-            try {
-                const decrypted = decryptBlob(
-                    fromBase64(entry.encryptedData),
-                    fromBase64(entry.decryptionHeader),
-                    key,
-                );
-                const jsonStr = gunzipSync(Buffer.from(decrypted)).toString(
-                    "utf-8",
-                );
-                result.set(entry.fileID, JSON.parse(jsonStr));
-            } catch {
-                // Corrupted ML data for this file; skip it
-            }
-        }
-    }
-    return result;
-};
-
 // Extract the raw EXIF APP1 segment from JPEG bytes. Returns the EXIF
 // data buffer (starting after the APP1 length field, at the "Exif\0\0"
 // header) or undefined if no APP1 marker is found.
@@ -228,8 +183,8 @@ export const runMetadataBackup = async (
     }
 
     log("Fetching ML data (face detections, CLIP embeddings)...");
-    const mlDataMap = await fetchMLDataForFiles(
-        client,
+    const mlDataMap = await fetchMLData(
+        client.getApiClient(),
         [...fileKeys.keys()],
         fileKeys,
     );